2024-11-16 04:23:47,912 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 04:23:47,926 main DEBUG Took 0.012319 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 04:23:47,927 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 04:23:47,927 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 04:23:47,928 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 04:23:47,929 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,937 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 04:23:47,949 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,950 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,951 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,952 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,954 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,955 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,957 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,959 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,960 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,961 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,962 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,963 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 04:23:47,964 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,964 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 04:23:47,966 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 04:23:47,968 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 04:23:47,971 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 04:23:47,972 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 04:23:47,974 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 04:23:47,974 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 04:23:47,984 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 04:23:47,987 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 04:23:47,989 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 04:23:47,990 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 04:23:47,990 main DEBUG createAppenders(={Console}) 2024-11-16 04:23:47,991 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 04:23:47,991 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 04:23:47,992 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 04:23:47,992 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 04:23:47,992 main DEBUG OutputStream closed 2024-11-16 04:23:47,993 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 04:23:47,993 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 04:23:47,993 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 04:23:48,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 04:23:48,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 04:23:48,083 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 04:23:48,085 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 04:23:48,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 04:23:48,086 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 04:23:48,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 04:23:48,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 04:23:48,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 04:23:48,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 04:23:48,088 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 04:23:48,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 04:23:48,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 04:23:48,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 04:23:48,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 04:23:48,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 04:23:48,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 04:23:48,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 04:23:48,094 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 04:23:48,094 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 04:23:48,094 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 04:23:48,095 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T04:23:48,355 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18 2024-11-16 04:23:48,358 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 04:23:48,358 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T04:23:48,365 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-16T04:23:48,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T04:23:48,399 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675, deleteOnExit=true 2024-11-16T04:23:48,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T04:23:48,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/test.cache.data in system properties and HBase conf 2024-11-16T04:23:48,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T04:23:48,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/hadoop.log.dir in system properties and HBase conf 2024-11-16T04:23:48,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T04:23:48,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T04:23:48,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T04:23:48,486 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T04:23:48,568 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T04:23:48,572 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T04:23:48,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T04:23:48,573 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T04:23:48,574 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T04:23:48,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T04:23:48,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T04:23:48,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T04:23:48,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T04:23:48,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T04:23:48,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/nfs.dump.dir in system properties and HBase conf 2024-11-16T04:23:48,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/java.io.tmpdir in system properties and HBase conf 2024-11-16T04:23:48,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T04:23:48,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T04:23:48,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T04:23:49,715 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T04:23:49,783 INFO [Time-limited test {}] log.Log(170): Logging initialized @2671ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T04:23:49,856 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T04:23:49,930 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T04:23:49,963 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T04:23:49,964 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T04:23:49,966 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T04:23:49,981 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T04:23:49,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/hadoop.log.dir/,AVAILABLE} 2024-11-16T04:23:49,985 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T04:23:50,179 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f7463d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/java.io.tmpdir/jetty-localhost-44379-hadoop-hdfs-3_4_1-tests_jar-_-any-11528664175188101316/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T04:23:50,193 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:44379} 2024-11-16T04:23:50,193 INFO [Time-limited test {}] server.Server(415): Started @3082ms 2024-11-16T04:23:50,932 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T04:23:50,946 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T04:23:50,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T04:23:50,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T04:23:50,954 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T04:23:50,956 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b40f5c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/hadoop.log.dir/,AVAILABLE} 2024-11-16T04:23:50,957 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1118b2a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T04:23:51,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3b60cd11{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/java.io.tmpdir/jetty-localhost-34377-hadoop-hdfs-3_4_1-tests_jar-_-any-6053468496684279902/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T04:23:51,079 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286a1f36{HTTP/1.1, (http/1.1)}{localhost:34377} 2024-11-16T04:23:51,079 INFO [Time-limited test {}] server.Server(415): Started @3968ms 2024-11-16T04:23:51,139 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T04:23:52,232 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/data/data1/current/BP-685687939-172.17.0.2-1731731029115/current, will proceed with Du for space computation calculation, 2024-11-16T04:23:52,233 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/data/data2/current/BP-685687939-172.17.0.2-1731731029115/current, will proceed with Du for space computation calculation, 2024-11-16T04:23:52,269 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T04:23:52,324 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd23b7d9ea60b5f4b with lease ID 0x493c1ac8c94af93c: Processing first storage report for DS-62e857fa-11d4-46cf-956c-167a5b8c3e21 from datanode DatanodeRegistration(127.0.0.1:46103, datanodeUuid=a61d9fca-c2f0-4b33-9fbc-13316ed550d7, infoPort=42871, infoSecurePort=0, ipcPort=35683, storageInfo=lv=-57;cid=testClusterID;nsid=1927150661;c=1731731029115) 2024-11-16T04:23:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd23b7d9ea60b5f4b with lease ID 0x493c1ac8c94af93c: from storage DS-62e857fa-11d4-46cf-956c-167a5b8c3e21 node DatanodeRegistration(127.0.0.1:46103, datanodeUuid=a61d9fca-c2f0-4b33-9fbc-13316ed550d7, infoPort=42871, infoSecurePort=0, ipcPort=35683, storageInfo=lv=-57;cid=testClusterID;nsid=1927150661;c=1731731029115), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T04:23:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd23b7d9ea60b5f4b with lease ID 0x493c1ac8c94af93c: Processing first storage report for DS-24a722d9-e59c-4a62-b037-252ee6f0f07d from datanode DatanodeRegistration(127.0.0.1:46103, datanodeUuid=a61d9fca-c2f0-4b33-9fbc-13316ed550d7, infoPort=42871, infoSecurePort=0, ipcPort=35683, storageInfo=lv=-57;cid=testClusterID;nsid=1927150661;c=1731731029115) 2024-11-16T04:23:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd23b7d9ea60b5f4b with lease ID 0x493c1ac8c94af93c: from storage DS-24a722d9-e59c-4a62-b037-252ee6f0f07d node DatanodeRegistration(127.0.0.1:46103, datanodeUuid=a61d9fca-c2f0-4b33-9fbc-13316ed550d7, infoPort=42871, infoSecurePort=0, ipcPort=35683, storageInfo=lv=-57;cid=testClusterID;nsid=1927150661;c=1731731029115), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T04:23:52,339 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18 2024-11-16T04:23:52,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/zookeeper_0, clientPort=60710, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T04:23:52,429 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60710 2024-11-16T04:23:52,439 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:52,442 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:52,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741825_1001 (size=7) 2024-11-16T04:23:53,079 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd with version=8 2024-11-16T04:23:53,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/hbase-staging 2024-11-16T04:23:53,200 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T04:23:53,461 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fda3efb342d8:0 server-side Connection retries=45 2024-11-16T04:23:53,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T04:23:53,471 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T04:23:53,476 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T04:23:53,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T04:23:53,477 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T04:23:53,615 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T04:23:53,671 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T04:23:53,681 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T04:23:53,685 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:23:53,710 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33114 (auto-detected) 2024-11-16T04:23:53,712 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T04:23:53,734 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45087 2024-11-16T04:23:53,760 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45087 connecting to ZooKeeper ensemble=127.0.0.1:60710 2024-11-16T04:23:53,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450870x0, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T04:23:53,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45087-0x10141e284fa0000 connected 2024-11-16T04:23:53,957 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:53,960 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:53,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T04:23:53,975 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd, hbase.cluster.distributed=false 2024-11-16T04:23:54,006 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T04:23:54,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45087 2024-11-16T04:23:54,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45087 2024-11-16T04:23:54,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45087 2024-11-16T04:23:54,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45087 2024-11-16T04:23:54,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45087 2024-11-16T04:23:54,125 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda3efb342d8:0 server-side Connection retries=45 2024-11-16T04:23:54,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T04:23:54,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T04:23:54,127 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T04:23:54,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T04:23:54,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T04:23:54,130 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T04:23:54,134 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T04:23:54,135 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45455 2024-11-16T04:23:54,138 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45455 connecting to ZooKeeper ensemble=127.0.0.1:60710 2024-11-16T04:23:54,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:54,143 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:54,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454550x0, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T04:23:54,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45455-0x10141e284fa0001 connected 2024-11-16T04:23:54,161 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T04:23:54,166 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T04:23:54,176 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T04:23:54,180 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T04:23:54,186 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T04:23:54,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45455 2024-11-16T04:23:54,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45455 2024-11-16T04:23:54,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45455 2024-11-16T04:23:54,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45455 2024-11-16T04:23:54,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45455 2024-11-16T04:23:54,216 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fda3efb342d8:45087 2024-11-16T04:23:54,217 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fda3efb342d8,45087,1731731033310 2024-11-16T04:23:54,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T04:23:54,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T04:23:54,237 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fda3efb342d8,45087,1731731033310 2024-11-16T04:23:54,276 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T04:23:54,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:54,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:54,279 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T04:23:54,281 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fda3efb342d8,45087,1731731033310 from backup master directory 2024-11-16T04:23:54,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fda3efb342d8,45087,1731731033310 2024-11-16T04:23:54,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T04:23:54,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T04:23:54,292 WARN [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T04:23:54,293 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fda3efb342d8,45087,1731731033310 2024-11-16T04:23:54,295 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T04:23:54,296 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T04:23:54,353 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/hbase.id] with ID: 8b208c1b-cf16-46a9-aa71-ed819631aff1 2024-11-16T04:23:54,353 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/.tmp/hbase.id 2024-11-16T04:23:54,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741826_1002 (size=42) 2024-11-16T04:23:54,766 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/.tmp/hbase.id]:[hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/hbase.id] 2024-11-16T04:23:54,817 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T04:23:54,822 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T04:23:54,839 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-16T04:23:54,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:54,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:54,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741827_1003 (size=196) 2024-11-16T04:23:55,336 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:23:55,339 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T04:23:55,353 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:23:55,356 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T04:23:55,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741828_1004 (size=1189) 2024-11-16T04:23:55,802 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store 2024-11-16T04:23:55,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741829_1005 (size=34) 2024-11-16T04:23:56,225 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T04:23:56,228 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:23:56,229 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T04:23:56,229 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:23:56,229 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:23:56,231 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T04:23:56,231 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:23:56,231 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:23:56,232 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731731036229Disabling compacts and flushes for region at 1731731036229Disabling writes for close at 1731731036231 (+2 ms)Writing region close event to WAL at 1731731036231Closed at 1731731036231 2024-11-16T04:23:56,234 WARN [master/fda3efb342d8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/.initializing 2024-11-16T04:23:56,234 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/WALs/fda3efb342d8,45087,1731731033310 2024-11-16T04:23:56,241 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T04:23:56,255 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda3efb342d8%2C45087%2C1731731033310, suffix=, logDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/WALs/fda3efb342d8,45087,1731731033310, archiveDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/oldWALs, maxLogs=10 2024-11-16T04:23:56,275 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/WALs/fda3efb342d8,45087,1731731033310/fda3efb342d8%2C45087%2C1731731033310.1731731036259, exclude list is [], retry=0 2024-11-16T04:23:56,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46103,DS-62e857fa-11d4-46cf-956c-167a5b8c3e21,DISK] 2024-11-16T04:23:56,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T04:23:56,324 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/WALs/fda3efb342d8,45087,1731731033310/fda3efb342d8%2C45087%2C1731731033310.1731731036259 2024-11-16T04:23:56,325 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42871:42871)] 2024-11-16T04:23:56,326 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:23:56,326 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:23:56,329 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,330 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,366 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,390 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T04:23:56,393 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:56,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:56,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,400 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T04:23:56,400 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:56,401 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:23:56,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T04:23:56,406 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:56,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:23:56,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,410 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T04:23:56,410 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:56,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:23:56,412 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,416 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,417 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,424 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,425 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,430 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T04:23:56,436 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T04:23:56,441 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:23:56,443 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69501040, jitterRate=0.03564620018005371}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T04:23:56,452 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731731036341Initializing all the Stores at 1731731036343 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731036343Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731036344 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731036344Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731036344Cleaning up temporary data from old regions at 1731731036425 (+81 ms)Region opened successfully at 1731731036452 (+27 ms) 2024-11-16T04:23:56,454 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T04:23:56,494 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d03a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda3efb342d8/172.17.0.2:0 2024-11-16T04:23:56,530 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T04:23:56,543 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T04:23:56,543 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T04:23:56,546 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T04:23:56,548 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T04:23:56,555 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T04:23:56,555 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T04:23:56,587 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T04:23:56,597 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T04:23:56,607 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T04:23:56,610 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T04:23:56,612 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T04:23:56,623 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T04:23:56,626 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T04:23:56,631 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T04:23:56,644 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T04:23:56,647 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T04:23:56,660 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T04:23:56,681 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T04:23:56,691 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T04:23:56,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T04:23:56,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T04:23:56,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:56,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:56,710 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fda3efb342d8,45087,1731731033310, sessionid=0x10141e284fa0000, setting cluster-up flag (Was=false) 2024-11-16T04:23:56,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:56,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:56,776 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T04:23:56,779 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda3efb342d8,45087,1731731033310 2024-11-16T04:23:56,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:56,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:56,834 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T04:23:56,836 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda3efb342d8,45087,1731731033310 2024-11-16T04:23:56,841 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T04:23:56,901 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(746): ClusterId : 8b208c1b-cf16-46a9-aa71-ed819631aff1 2024-11-16T04:23:56,904 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T04:23:56,915 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T04:23:56,920 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T04:23:56,920 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T04:23:56,928 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T04:23:56,935 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T04:23:56,935 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T04:23:56,936 DEBUG [RS:0;fda3efb342d8:45455 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ecbdef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda3efb342d8/172.17.0.2:0 2024-11-16T04:23:56,942 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fda3efb342d8,45087,1731731033310 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T04:23:56,950 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fda3efb342d8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T04:23:56,951 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fda3efb342d8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T04:23:56,951 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fda3efb342d8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T04:23:56,951 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fda3efb342d8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T04:23:56,951 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fda3efb342d8:0, corePoolSize=10, maxPoolSize=10 2024-11-16T04:23:56,952 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:56,952 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fda3efb342d8:0, corePoolSize=2, maxPoolSize=2 2024-11-16T04:23:56,952 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:56,954 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fda3efb342d8:45455 2024-11-16T04:23:56,955 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731731066955 2024-11-16T04:23:56,957 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T04:23:56,958 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T04:23:56,958 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T04:23:56,958 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T04:23:56,958 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T04:23:56,959 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T04:23:56,959 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T04:23:56,962 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda3efb342d8,45087,1731731033310 with port=45455, startcode=1731731034087 2024-11-16T04:23:56,962 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T04:23:56,962 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T04:23:56,963 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T04:23:56,963 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T04:23:56,964 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:56,966 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:56,966 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T04:23:56,969 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T04:23:56,970 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T04:23:56,971 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T04:23:56,973 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T04:23:56,974 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T04:23:56,976 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fda3efb342d8:0:becomeActiveMaster-HFileCleaner.large.0-1731731036975,5,FailOnTimeoutGroup] 2024-11-16T04:23:56,977 DEBUG [RS:0;fda3efb342d8:45455 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T04:23:56,980 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fda3efb342d8:0:becomeActiveMaster-HFileCleaner.small.0-1731731036976,5,FailOnTimeoutGroup] 2024-11-16T04:23:56,980 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:56,980 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T04:23:56,981 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:56,982 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741831_1007 (size=1321) 2024-11-16T04:23:56,993 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T04:23:56,994 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:23:57,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741832_1008 (size=32) 2024-11-16T04:23:57,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34329, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T04:23:57,026 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45087 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45087 {}] master.ServerManager(517): Registering regionserver=fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,046 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:23:57,046 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43469 2024-11-16T04:23:57,046 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T04:23:57,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T04:23:57,060 DEBUG [RS:0;fda3efb342d8:45455 {}] zookeeper.ZKUtil(111): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,060 WARN [RS:0;fda3efb342d8:45455 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T04:23:57,061 INFO [RS:0;fda3efb342d8:45455 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T04:23:57,061 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,064 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda3efb342d8,45455,1731731034087] 2024-11-16T04:23:57,088 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T04:23:57,106 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T04:23:57,113 INFO [RS:0;fda3efb342d8:45455 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T04:23:57,113 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,114 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T04:23:57,122 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T04:23:57,125 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,125 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,126 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,126 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,126 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,127 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,127 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda3efb342d8:0, corePoolSize=2, maxPoolSize=2 2024-11-16T04:23:57,127 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,127 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,127 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,128 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,128 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,128 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda3efb342d8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T04:23:57,128 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda3efb342d8:0, corePoolSize=3, maxPoolSize=3 2024-11-16T04:23:57,129 DEBUG [RS:0;fda3efb342d8:45455 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0, corePoolSize=3, maxPoolSize=3 2024-11-16T04:23:57,130 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,130 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,131 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,131 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,131 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,131 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45455,1731731034087-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T04:23:57,152 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T04:23:57,154 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45455,1731731034087-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,154 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,154 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.Replication(171): fda3efb342d8,45455,1731731034087 started 2024-11-16T04:23:57,174 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:57,174 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1482): Serving as fda3efb342d8,45455,1731731034087, RpcServer on fda3efb342d8/172.17.0.2:45455, sessionid=0x10141e284fa0001 2024-11-16T04:23:57,175 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T04:23:57,175 DEBUG [RS:0;fda3efb342d8:45455 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,175 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda3efb342d8,45455,1731731034087' 2024-11-16T04:23:57,175 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T04:23:57,176 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T04:23:57,177 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T04:23:57,177 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T04:23:57,177 DEBUG [RS:0;fda3efb342d8:45455 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,178 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda3efb342d8,45455,1731731034087' 2024-11-16T04:23:57,178 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T04:23:57,179 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T04:23:57,179 DEBUG [RS:0;fda3efb342d8:45455 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T04:23:57,180 INFO [RS:0;fda3efb342d8:45455 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T04:23:57,180 INFO [RS:0;fda3efb342d8:45455 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T04:23:57,286 INFO [RS:0;fda3efb342d8:45455 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T04:23:57,290 INFO [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda3efb342d8%2C45455%2C1731731034087, suffix=, logDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087, archiveDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/oldWALs, maxLogs=32 2024-11-16T04:23:57,305 DEBUG [RS:0;fda3efb342d8:45455 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087/fda3efb342d8%2C45455%2C1731731034087.1731731037292, exclude list is [], retry=0 2024-11-16T04:23:57,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46103,DS-62e857fa-11d4-46cf-956c-167a5b8c3e21,DISK] 2024-11-16T04:23:57,314 INFO [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087/fda3efb342d8%2C45455%2C1731731034087.1731731037292 2024-11-16T04:23:57,315 DEBUG [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42871:42871)] 2024-11-16T04:23:57,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:23:57,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T04:23:57,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T04:23:57,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,429 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,429 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T04:23:57,433 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T04:23:57,433 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T04:23:57,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T04:23:57,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T04:23:57,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T04:23:57,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T04:23:57,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740 2024-11-16T04:23:57,445 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740 2024-11-16T04:23:57,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T04:23:57,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T04:23:57,449 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:23:57,451 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T04:23:57,455 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:23:57,456 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68484380, jitterRate=0.020496785640716553}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:23:57,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731731037421Initializing all the Stores at 1731731037424 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731037424Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731037424Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731037424Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731037424Cleaning up temporary data from old regions at 1731731037448 (+24 ms)Region opened successfully at 1731731037458 (+10 ms) 2024-11-16T04:23:57,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T04:23:57,458 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T04:23:57,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T04:23:57,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T04:23:57,458 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T04:23:57,460 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T04:23:57,460 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731731037458Disabling compacts and flushes for region at 1731731037458Disabling writes for close at 1731731037458Writing region close event to WAL at 1731731037459 (+1 ms)Closed at 1731731037460 (+1 ms) 2024-11-16T04:23:57,463 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T04:23:57,464 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T04:23:57,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T04:23:57,475 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T04:23:57,478 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T04:23:57,631 DEBUG [fda3efb342d8:45087 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T04:23:57,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:23:57,650 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda3efb342d8,45455,1731731034087, state=OPENING 2024-11-16T04:23:57,707 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T04:23:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:57,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:23:57,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T04:23:57,719 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T04:23:57,721 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T04:23:57,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:23:57,895 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T04:23:57,899 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55523, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T04:23:57,910 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T04:23:57,911 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T04:23:57,912 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T04:23:57,915 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda3efb342d8%2C45455%2C1731731034087.meta, suffix=.meta, logDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087, archiveDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/oldWALs, maxLogs=32 2024-11-16T04:23:57,929 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087/fda3efb342d8%2C45455%2C1731731034087.meta.1731731037917.meta, exclude list is [], retry=0 2024-11-16T04:23:57,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46103,DS-62e857fa-11d4-46cf-956c-167a5b8c3e21,DISK] 2024-11-16T04:23:57,936 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/WALs/fda3efb342d8,45455,1731731034087/fda3efb342d8%2C45455%2C1731731034087.meta.1731731037917.meta 2024-11-16T04:23:57,937 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42871:42871)] 2024-11-16T04:23:57,937 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:23:57,939 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T04:23:57,941 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T04:23:57,946 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T04:23:57,950 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T04:23:57,951 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:23:57,951 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T04:23:57,952 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T04:23:57,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T04:23:57,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T04:23:57,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T04:23:57,960 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T04:23:57,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T04:23:57,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T04:23:57,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T04:23:57,966 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T04:23:57,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:57,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T04:23:57,967 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T04:23:57,969 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740 2024-11-16T04:23:57,973 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740 2024-11-16T04:23:57,975 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T04:23:57,976 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T04:23:57,977 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:23:57,980 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T04:23:57,982 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67402941, jitterRate=0.004382088780403137}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:23:57,982 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T04:23:57,985 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731731037952Writing region info on filesystem at 1731731037952Initializing all the Stores at 1731731037954 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731037954Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731037955 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731037955Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731731037955Cleaning up temporary data from old regions at 1731731037976 (+21 ms)Running coprocessor post-open hooks at 1731731037983 (+7 ms)Region opened successfully at 1731731037984 (+1 ms) 2024-11-16T04:23:57,992 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731731037888 2024-11-16T04:23:58,004 DEBUG [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T04:23:58,005 INFO [RS_OPEN_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T04:23:58,006 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:23:58,009 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda3efb342d8,45455,1731731034087, state=OPEN 2024-11-16T04:23:58,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T04:23:58,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T04:23:58,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T04:23:58,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T04:23:58,099 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:23:58,110 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T04:23:58,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda3efb342d8,45455,1731731034087 in 376 msec 2024-11-16T04:23:58,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T04:23:58,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 645 msec 2024-11-16T04:23:58,123 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T04:23:58,124 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T04:23:58,150 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:23:58,152 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:23:58,178 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:23:58,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59743, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:23:58,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3340 sec 2024-11-16T04:23:58,208 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731731038208, completionTime=-1 2024-11-16T04:23:58,212 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T04:23:58,212 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T04:23:58,250 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-16T04:23:58,250 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731731098250 2024-11-16T04:23:58,250 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731731158250 2024-11-16T04:23:58,250 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 38 msec 2024-11-16T04:23:58,253 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45087,1731731033310-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,253 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45087,1731731033310-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,253 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45087,1731731033310-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,255 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fda3efb342d8:45087, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,255 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,256 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,264 DEBUG [master/fda3efb342d8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T04:23:58,291 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.998sec 2024-11-16T04:23:58,292 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T04:23:58,293 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T04:23:58,294 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T04:23:58,295 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T04:23:58,295 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T04:23:58,296 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45087,1731731033310-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T04:23:58,296 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45087,1731731033310-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T04:23:58,305 DEBUG [master/fda3efb342d8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T04:23:58,306 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T04:23:58,307 INFO [master/fda3efb342d8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda3efb342d8,45087,1731731033310-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T04:23:58,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531476d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:23:58,315 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T04:23:58,315 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T04:23:58,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:23:58,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:23:58,329 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:23:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:23:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:23:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c86253f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:23:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:23:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:23:58,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:23:58,344 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:23:58,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580c843b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:23:58,348 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:23:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:23:58,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:23:58,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:23:58,384 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fda3efb342d8,45087,1731731033310 2024-11-16T04:23:58,395 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=344, ProcessCount=11, AvailableMemoryMB=9331 2024-11-16T04:23:58,403 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:23:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:23:58,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@571ad612 2024-11-16T04:23:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:23:58,464 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:23:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:23:58,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:23:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T04:23:58,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T04:23:58,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T04:23:58,499 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:58,504 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T04:23:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T04:23:58,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741835_1011 (size=959) 2024-11-16T04:23:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T04:23:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T04:23:58,936 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:23:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741836_1012 (size=53) 2024-11-16T04:23:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T04:23:59,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:23:59,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 20bf25e458f1c8ea7c07f170b9f8e62a, disabling compactions & flushes 2024-11-16T04:23:59,348 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,348 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. after waiting 0 ms 2024-11-16T04:23:59,349 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,349 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,349 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 20bf25e458f1c8ea7c07f170b9f8e62a: Waiting for close lock at 1731731039348Disabling compacts and flushes for region at 1731731039348Disabling writes for close at 1731731039348Writing region close event to WAL at 1731731039349 (+1 ms)Closed at 1731731039349 2024-11-16T04:23:59,352 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T04:23:59,359 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731731039352"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731731039352"}]},"ts":"1731731039352"} 2024-11-16T04:23:59,364 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T04:23:59,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T04:23:59,369 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731039366"}]},"ts":"1731731039366"} 2024-11-16T04:23:59,373 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T04:23:59,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, ASSIGN}] 2024-11-16T04:23:59,378 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, ASSIGN 2024-11-16T04:23:59,379 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, ASSIGN; state=OFFLINE, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=false 2024-11-16T04:23:59,532 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=20bf25e458f1c8ea7c07f170b9f8e62a, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:23:59,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, ASSIGN because future has completed 2024-11-16T04:23:59,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:23:59,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T04:23:59,705 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,705 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:23:59,706 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,706 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:23:59,706 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,706 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,709 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,712 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20bf25e458f1c8ea7c07f170b9f8e62a columnFamilyName A 2024-11-16T04:23:59,712 DEBUG [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:59,714 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] regionserver.HStore(327): Store=20bf25e458f1c8ea7c07f170b9f8e62a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:23:59,714 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,716 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20bf25e458f1c8ea7c07f170b9f8e62a columnFamilyName B 2024-11-16T04:23:59,717 DEBUG [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:59,717 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] regionserver.HStore(327): Store=20bf25e458f1c8ea7c07f170b9f8e62a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:23:59,718 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,720 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20bf25e458f1c8ea7c07f170b9f8e62a columnFamilyName C 2024-11-16T04:23:59,720 DEBUG [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:23:59,721 INFO [StoreOpener-20bf25e458f1c8ea7c07f170b9f8e62a-1 {}] regionserver.HStore(327): Store=20bf25e458f1c8ea7c07f170b9f8e62a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:23:59,721 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,722 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,723 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,725 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,725 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,726 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:23:59,728 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,732 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:23:59,733 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 20bf25e458f1c8ea7c07f170b9f8e62a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63450089, jitterRate=-0.05451999604701996}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:23:59,733 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:23:59,734 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 20bf25e458f1c8ea7c07f170b9f8e62a: Running coprocessor pre-open hook at 1731731039707Writing region info on filesystem at 1731731039707Initializing all the Stores at 1731731039708 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731039708Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731039709 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731039709Cleaning up temporary data from old regions at 1731731039725 (+16 ms)Running coprocessor post-open hooks at 1731731039733 (+8 ms)Region opened successfully at 1731731039734 (+1 ms) 2024-11-16T04:23:59,736 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., pid=6, masterSystemTime=1731731039697 2024-11-16T04:23:59,740 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,740 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:23:59,741 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=20bf25e458f1c8ea7c07f170b9f8e62a, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:23:59,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:23:59,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T04:23:59,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 in 204 msec 2024-11-16T04:23:59,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T04:23:59,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, ASSIGN in 376 msec 2024-11-16T04:23:59,757 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T04:23:59,757 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731039757"}]},"ts":"1731731039757"} 2024-11-16T04:23:59,761 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T04:23:59,762 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T04:23:59,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2770 sec 2024-11-16T04:24:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T04:24:00,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:00,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ccc16a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,680 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8448541, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,685 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a646f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,687 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,694 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:24:00,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2bd4bf8e 2024-11-16T04:24:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:24:00,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:24:00,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1b6764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,706 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2886558d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,710 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae98662, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,713 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,716 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1b920f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,723 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@223b8369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,727 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1bd73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3cf0fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75122da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d96214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,736 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f776e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19c4894f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,739 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d6e4493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,742 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e1b379, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621a7b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,746 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57865cab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,748 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf8fae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3484bd27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21338ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,753 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@585eb20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6837d2cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:00,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:00,758 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:00,758 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56225fcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:00,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:00,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7b74c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:00,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:00,768 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,768 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,768 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,770 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,773 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,775 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T04:24:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T04:24:00,782 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,784 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:00,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,788 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:00,796 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,801 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,803 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,805 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:00,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:00,810 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:24:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:00,864 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:00,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:24:00,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T04:24:00,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:00,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:00,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:00,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/6c4c8ab912b24d2c81b9fb5e25319acb is 50, key is test_row_0/A:col10/1731731040832/Put/seqid=0 2024-11-16T04:24:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741837_1013 (size=14341) 2024-11-16T04:24:01,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/6c4c8ab912b24d2c81b9fb5e25319acb 2024-11-16T04:24:01,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731101049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731101042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731101060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731101071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731101075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T04:24:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/b97e529fd62a465297dda59615aea769 is 50, key is test_row_0/B:col10/1731731040832/Put/seqid=0 2024-11-16T04:24:01,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741838_1014 (size=12001) 2024-11-16T04:24:01,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/b97e529fd62a465297dda59615aea769 2024-11-16T04:24:01,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731101215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731101215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731101216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731101216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731101217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/093ca87cca7e426f9d560c298a9b146d is 50, key is test_row_0/C:col10/1731731040832/Put/seqid=0 2024-11-16T04:24:01,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741839_1015 (size=12001) 2024-11-16T04:24:01,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/093ca87cca7e426f9d560c298a9b146d 2024-11-16T04:24:01,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/6c4c8ab912b24d2c81b9fb5e25319acb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6c4c8ab912b24d2c81b9fb5e25319acb 2024-11-16T04:24:01,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6c4c8ab912b24d2c81b9fb5e25319acb, entries=200, sequenceid=13, filesize=14.0 K 2024-11-16T04:24:01,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/b97e529fd62a465297dda59615aea769 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b97e529fd62a465297dda59615aea769 2024-11-16T04:24:01,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b97e529fd62a465297dda59615aea769, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T04:24:01,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/093ca87cca7e426f9d560c298a9b146d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/093ca87cca7e426f9d560c298a9b146d 2024-11-16T04:24:01,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/093ca87cca7e426f9d560c298a9b146d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T04:24:01,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 20bf25e458f1c8ea7c07f170b9f8e62a in 457ms, sequenceid=13, compaction requested=false 2024-11-16T04:24:01,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:01,328 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T04:24:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:01,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T04:24:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:01,431 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:01,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T04:24:01,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/5f840beffacb45618483a0807fb205f3 is 50, key is test_row_0/A:col10/1731731041429/Put/seqid=0 2024-11-16T04:24:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731101460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731101459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741840_1016 (size=14341) 2024-11-16T04:24:01,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/5f840beffacb45618483a0807fb205f3 2024-11-16T04:24:01,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731101471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731101471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731101471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/ac70378ec22f4d2c839ff5aa45a9f05b is 50, key is test_row_0/B:col10/1731731041429/Put/seqid=0 2024-11-16T04:24:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741841_1017 (size=12001) 2024-11-16T04:24:01,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/ac70378ec22f4d2c839ff5aa45a9f05b 2024-11-16T04:24:01,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c04f841a361c43e9b866d24e9dc01328 is 50, key is test_row_0/C:col10/1731731041429/Put/seqid=0 2024-11-16T04:24:01,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731101578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731101579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731101594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731101595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731101595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741842_1018 (size=12001) 2024-11-16T04:24:01,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c04f841a361c43e9b866d24e9dc01328 2024-11-16T04:24:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/5f840beffacb45618483a0807fb205f3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/5f840beffacb45618483a0807fb205f3 2024-11-16T04:24:01,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/5f840beffacb45618483a0807fb205f3, entries=200, sequenceid=40, filesize=14.0 K 2024-11-16T04:24:01,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/ac70378ec22f4d2c839ff5aa45a9f05b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ac70378ec22f4d2c839ff5aa45a9f05b 2024-11-16T04:24:01,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:01,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:01,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ac70378ec22f4d2c839ff5aa45a9f05b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T04:24:01,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c04f841a361c43e9b866d24e9dc01328 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c04f841a361c43e9b866d24e9dc01328 2024-11-16T04:24:01,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c04f841a361c43e9b866d24e9dc01328, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T04:24:01,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=53.67 KB/54960 for 20bf25e458f1c8ea7c07f170b9f8e62a in 261ms, sequenceid=40, compaction requested=false 2024-11-16T04:24:01,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:01,799 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:01,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:24:01,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:01,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/1a1cc6d526e1452789f1e36c43224d85 is 50, key is test_row_0/A:col10/1731731041796/Put/seqid=0 2024-11-16T04:24:01,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741843_1019 (size=12001) 2024-11-16T04:24:01,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731101867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731101867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731101870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731101874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731101877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T04:24:01,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:01,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:01,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:01,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:01,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731101985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731101985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:01,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731101995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731101995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731101995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:02,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:02,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:02,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731102194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731102195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731102204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731102204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731102215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/1a1cc6d526e1452789f1e36c43224d85 2024-11-16T04:24:02,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/d934bc5225364538b21ab0f88c1363ba is 50, key is test_row_0/B:col10/1731731041796/Put/seqid=0 2024-11-16T04:24:02,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:02,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:02,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741844_1020 (size=12001) 2024-11-16T04:24:02,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/d934bc5225364538b21ab0f88c1363ba 2024-11-16T04:24:02,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/44a9b3e4042440f094b8133805d6f172 is 50, key is test_row_0/C:col10/1731731041796/Put/seqid=0 2024-11-16T04:24:02,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741845_1021 (size=12001) 2024-11-16T04:24:02,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/44a9b3e4042440f094b8133805d6f172 2024-11-16T04:24:02,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/1a1cc6d526e1452789f1e36c43224d85 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1a1cc6d526e1452789f1e36c43224d85 2024-11-16T04:24:02,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1a1cc6d526e1452789f1e36c43224d85, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T04:24:02,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/d934bc5225364538b21ab0f88c1363ba as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d934bc5225364538b21ab0f88c1363ba 2024-11-16T04:24:02,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d934bc5225364538b21ab0f88c1363ba, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T04:24:02,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/44a9b3e4042440f094b8133805d6f172 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/44a9b3e4042440f094b8133805d6f172 2024-11-16T04:24:02,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/44a9b3e4042440f094b8133805d6f172, entries=150, sequenceid=52, filesize=11.7 K 2024-11-16T04:24:02,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 20bf25e458f1c8ea7c07f170b9f8e62a in 618ms, sequenceid=52, compaction requested=true 2024-11-16T04:24:02,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:02,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:02,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:02,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:02,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:02,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:02,424 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:02,424 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:02,427 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:02,430 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:02,431 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,431 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b97e529fd62a465297dda59615aea769, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ac70378ec22f4d2c839ff5aa45a9f05b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d934bc5225364538b21ab0f88c1363ba] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.2 K 2024-11-16T04:24:02,433 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b97e529fd62a465297dda59615aea769, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731040832 2024-11-16T04:24:02,435 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ac70378ec22f4d2c839ff5aa45a9f05b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731731041028 2024-11-16T04:24:02,437 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:02,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T04:24:02,438 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:02,438 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,438 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6c4c8ab912b24d2c81b9fb5e25319acb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/5f840beffacb45618483a0807fb205f3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1a1cc6d526e1452789f1e36c43224d85] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=39.7 K 2024-11-16T04:24:02,438 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:24:02,440 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c4c8ab912b24d2c81b9fb5e25319acb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731040832 2024-11-16T04:24:02,442 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f840beffacb45618483a0807fb205f3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731731041028 2024-11-16T04:24:02,442 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d934bc5225364538b21ab0f88c1363ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731731041465 2024-11-16T04:24:02,443 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1cc6d526e1452789f1e36c43224d85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731731041465 2024-11-16T04:24:02,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/01c67b74f18048d4baadf398c6c0f5fc is 50, key is test_row_0/A:col10/1731731041855/Put/seqid=0 2024-11-16T04:24:02,493 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:02,494 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/5bd7b7e61ebc419297bf72f58acc7ea1 is 50, key is test_row_0/B:col10/1731731041796/Put/seqid=0 2024-11-16T04:24:02,500 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:02,501 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/001c0f9ed7a84019a398ee2553b5a4c0 is 50, key is test_row_0/A:col10/1731731041796/Put/seqid=0 2024-11-16T04:24:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:02,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:02,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741847_1023 (size=12104) 2024-11-16T04:24:02,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741846_1022 (size=12001) 2024-11-16T04:24:02,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/01c67b74f18048d4baadf398c6c0f5fc 2024-11-16T04:24:02,545 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/5bd7b7e61ebc419297bf72f58acc7ea1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5bd7b7e61ebc419297bf72f58acc7ea1 2024-11-16T04:24:02,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731102534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731102544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731102549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731102551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731102553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9283ba3f9e024b67b52cf9b3dfb56a46 is 50, key is test_row_0/B:col10/1731731041855/Put/seqid=0 2024-11-16T04:24:02,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741848_1024 (size=12104) 2024-11-16T04:24:02,582 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 5bd7b7e61ebc419297bf72f58acc7ea1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:02,583 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:02,585 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731042421; duration=0sec 2024-11-16T04:24:02,588 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:02,588 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:02,589 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:02,591 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:02,592 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:02,592 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,592 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/093ca87cca7e426f9d560c298a9b146d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c04f841a361c43e9b866d24e9dc01328, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/44a9b3e4042440f094b8133805d6f172] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.2 K 2024-11-16T04:24:02,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741849_1025 (size=12001) 2024-11-16T04:24:02,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9283ba3f9e024b67b52cf9b3dfb56a46 2024-11-16T04:24:02,595 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 093ca87cca7e426f9d560c298a9b146d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731040832 2024-11-16T04:24:02,612 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c04f841a361c43e9b866d24e9dc01328, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731731041028 2024-11-16T04:24:02,613 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 44a9b3e4042440f094b8133805d6f172, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731731041465 2024-11-16T04:24:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/ec2c5ddbf6914123942a4ca6d196c72b is 50, key is test_row_0/C:col10/1731731041855/Put/seqid=0 2024-11-16T04:24:02,638 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:02,639 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c599bfb980ea4912bd4b7d09ad532fe7 is 50, key is test_row_0/C:col10/1731731041796/Put/seqid=0 2024-11-16T04:24:02,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741850_1026 (size=12001) 2024-11-16T04:24:02,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/ec2c5ddbf6914123942a4ca6d196c72b 2024-11-16T04:24:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/01c67b74f18048d4baadf398c6c0f5fc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/01c67b74f18048d4baadf398c6c0f5fc 2024-11-16T04:24:02,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731102663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731102674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731102674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731102674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/01c67b74f18048d4baadf398c6c0f5fc, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T04:24:02,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741851_1027 (size=12104) 2024-11-16T04:24:02,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9283ba3f9e024b67b52cf9b3dfb56a46 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9283ba3f9e024b67b52cf9b3dfb56a46 2024-11-16T04:24:02,709 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c599bfb980ea4912bd4b7d09ad532fe7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c599bfb980ea4912bd4b7d09ad532fe7 2024-11-16T04:24:02,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9283ba3f9e024b67b52cf9b3dfb56a46, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T04:24:02,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/ec2c5ddbf6914123942a4ca6d196c72b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/ec2c5ddbf6914123942a4ca6d196c72b 2024-11-16T04:24:02,727 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into c599bfb980ea4912bd4b7d09ad532fe7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:02,727 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:02,727 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731042422; duration=0sec 2024-11-16T04:24:02,728 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:02,728 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:02,740 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/ec2c5ddbf6914123942a4ca6d196c72b, entries=150, sequenceid=76, filesize=11.7 K 2024-11-16T04:24:02,742 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 20bf25e458f1c8ea7c07f170b9f8e62a in 304ms, sequenceid=76, compaction requested=false 2024-11-16T04:24:02,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:02,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:02,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T04:24:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T04:24:02,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T04:24:02,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-16T04:24:02,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9850 sec 2024-11-16T04:24:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:02,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:02,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T04:24:02,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/4454592c001c42b9b6e09bdb4e03b621 is 50, key is test_row_0/A:col10/1731731042885/Put/seqid=0 2024-11-16T04:24:02,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741852_1028 (size=12001) 2024-11-16T04:24:02,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/4454592c001c42b9b6e09bdb4e03b621 2024-11-16T04:24:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T04:24:02,936 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:02,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T04:24:02,942 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T04:24:02,943 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:02,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:02,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731102936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8ac735a5db324aa2bac25834c221f6f2 is 50, key is test_row_0/B:col10/1731731042885/Put/seqid=0 2024-11-16T04:24:02,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731102945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731102945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731102947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:02,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:02,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741853_1029 (size=12001) 2024-11-16T04:24:02,985 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/001c0f9ed7a84019a398ee2553b5a4c0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/001c0f9ed7a84019a398ee2553b5a4c0 2024-11-16T04:24:02,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8ac735a5db324aa2bac25834c221f6f2 2024-11-16T04:24:03,007 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into 001c0f9ed7a84019a398ee2553b5a4c0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:03,007 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:03,008 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731042419; duration=0sec 2024-11-16T04:24:03,008 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:03,008 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:03,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/49f1e4f39464419580bfcffe29508d88 is 50, key is test_row_0/C:col10/1731731042885/Put/seqid=0 2024-11-16T04:24:03,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741854_1030 (size=12001) 2024-11-16T04:24:03,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/49f1e4f39464419580bfcffe29508d88 2024-11-16T04:24:03,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/4454592c001c42b9b6e09bdb4e03b621 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/4454592c001c42b9b6e09bdb4e03b621 2024-11-16T04:24:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T04:24:03,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/4454592c001c42b9b6e09bdb4e03b621, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T04:24:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731103055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731103064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8ac735a5db324aa2bac25834c221f6f2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ac735a5db324aa2bac25834c221f6f2 2024-11-16T04:24:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731103064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731103064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731103064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ac735a5db324aa2bac25834c221f6f2, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T04:24:03,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/49f1e4f39464419580bfcffe29508d88 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/49f1e4f39464419580bfcffe29508d88 2024-11-16T04:24:03,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T04:24:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/49f1e4f39464419580bfcffe29508d88, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T04:24:03,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 20bf25e458f1c8ea7c07f170b9f8e62a in 307ms, sequenceid=94, compaction requested=true 2024-11-16T04:24:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:03,197 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:03,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:03,197 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:03,199 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:03,199 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:03,199 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,200 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:03,200 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:03,200 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/001c0f9ed7a84019a398ee2553b5a4c0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/01c67b74f18048d4baadf398c6c0f5fc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/4454592c001c42b9b6e09bdb4e03b621] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.3 K 2024-11-16T04:24:03,200 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,200 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5bd7b7e61ebc419297bf72f58acc7ea1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9283ba3f9e024b67b52cf9b3dfb56a46, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ac735a5db324aa2bac25834c221f6f2] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.3 K 2024-11-16T04:24:03,201 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd7b7e61ebc419297bf72f58acc7ea1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731731041465 2024-11-16T04:24:03,202 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9283ba3f9e024b67b52cf9b3dfb56a46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731731041855 2024-11-16T04:24:03,202 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001c0f9ed7a84019a398ee2553b5a4c0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731731041465 2024-11-16T04:24:03,203 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ac735a5db324aa2bac25834c221f6f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731042885 2024-11-16T04:24:03,203 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c67b74f18048d4baadf398c6c0f5fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731731041855 2024-11-16T04:24:03,206 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4454592c001c42b9b6e09bdb4e03b621, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731042885 2024-11-16T04:24:03,225 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:03,226 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/21a1756515a0488889ec1d7c30c5c6a0 is 50, key is test_row_0/B:col10/1731731042885/Put/seqid=0 2024-11-16T04:24:03,235 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:03,236 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/41d58421ca7842c7b4c0969338c6b39a is 50, key is test_row_0/A:col10/1731731042885/Put/seqid=0 2024-11-16T04:24:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741855_1031 (size=12207) 2024-11-16T04:24:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T04:24:03,266 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/21a1756515a0488889ec1d7c30c5c6a0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/21a1756515a0488889ec1d7c30c5c6a0 2024-11-16T04:24:03,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741856_1032 (size=12207) 2024-11-16T04:24:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:03,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:03,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T04:24:03,293 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/41d58421ca7842c7b4c0969338c6b39a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/41d58421ca7842c7b4c0969338c6b39a 2024-11-16T04:24:03,295 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 21a1756515a0488889ec1d7c30c5c6a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:03,295 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:03,295 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731043197; duration=0sec 2024-11-16T04:24:03,295 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:03,295 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:03,296 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:03,298 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:03,298 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:03,298 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,299 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c599bfb980ea4912bd4b7d09ad532fe7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/ec2c5ddbf6914123942a4ca6d196c72b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/49f1e4f39464419580bfcffe29508d88] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.3 K 2024-11-16T04:24:03,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/0262e8f85d7d458a82601ab9d4cee9ea is 50, key is test_row_0/A:col10/1731731043279/Put/seqid=0 2024-11-16T04:24:03,300 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c599bfb980ea4912bd4b7d09ad532fe7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731731041465 2024-11-16T04:24:03,301 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ec2c5ddbf6914123942a4ca6d196c72b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731731041855 2024-11-16T04:24:03,302 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f1e4f39464419580bfcffe29508d88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731042885 2024-11-16T04:24:03,308 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into 41d58421ca7842c7b4c0969338c6b39a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:03,308 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:03,308 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731043196; duration=0sec 2024-11-16T04:24:03,308 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:03,308 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:03,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741857_1033 (size=16681) 2024-11-16T04:24:03,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/0262e8f85d7d458a82601ab9d4cee9ea 2024-11-16T04:24:03,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731103327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T04:24:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/1dc198a5b6464449962c8691fd74fa85 is 50, key is test_row_0/B:col10/1731731043279/Put/seqid=0 2024-11-16T04:24:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731103337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731103349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731103338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,384 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:03,385 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c7757a57dee3427c910da103e1b92337 is 50, key is test_row_0/C:col10/1731731042885/Put/seqid=0 2024-11-16T04:24:03,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741859_1035 (size=12207) 2024-11-16T04:24:03,425 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c7757a57dee3427c910da103e1b92337 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c7757a57dee3427c910da103e1b92337 2024-11-16T04:24:03,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741858_1034 (size=12001) 2024-11-16T04:24:03,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/1dc198a5b6464449962c8691fd74fa85 2024-11-16T04:24:03,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731103444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,455 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into c7757a57dee3427c910da103e1b92337(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:03,455 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:03,455 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731043197; duration=0sec 2024-11-16T04:24:03,456 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:03,456 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:03,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/92990378d3ac406aaf6b1aa1d4cf63f5 is 50, key is test_row_0/C:col10/1731731043279/Put/seqid=0 2024-11-16T04:24:03,464 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:24:03,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731103466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731103466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741860_1036 (size=12001) 2024-11-16T04:24:03,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/92990378d3ac406aaf6b1aa1d4cf63f5 2024-11-16T04:24:03,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T04:24:03,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:03,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:03,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/0262e8f85d7d458a82601ab9d4cee9ea as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0262e8f85d7d458a82601ab9d4cee9ea 2024-11-16T04:24:03,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0262e8f85d7d458a82601ab9d4cee9ea, entries=250, sequenceid=117, filesize=16.3 K 2024-11-16T04:24:03,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/1dc198a5b6464449962c8691fd74fa85 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1dc198a5b6464449962c8691fd74fa85 2024-11-16T04:24:03,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1dc198a5b6464449962c8691fd74fa85, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T04:24:03,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/92990378d3ac406aaf6b1aa1d4cf63f5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/92990378d3ac406aaf6b1aa1d4cf63f5 2024-11-16T04:24:03,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/92990378d3ac406aaf6b1aa1d4cf63f5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-16T04:24:03,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 20bf25e458f1c8ea7c07f170b9f8e62a in 276ms, sequenceid=117, compaction requested=false 2024-11-16T04:24:03,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T04:24:03,600 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T04:24:03,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T04:24:03,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:03,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:03,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T04:24:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:03,668 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T04:24:03,669 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T04:24:03,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/a48ff5b3ba05449db218e87f1040d899 is 50, key is test_row_0/A:col10/1731731043335/Put/seqid=0 2024-11-16T04:24:03,671 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T04:24:03,671 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T04:24:03,672 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T04:24:03,672 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T04:24:03,677 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T04:24:03,677 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T04:24:03,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741861_1037 (size=14541) 2024-11-16T04:24:03,718 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/a48ff5b3ba05449db218e87f1040d899 2024-11-16T04:24:03,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731103729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731103732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731103736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731103739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/a2438086290d450bb8f059396bfb765e is 50, key is test_row_0/B:col10/1731731043335/Put/seqid=0 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741862_1038 (size=12151) 2024-11-16T04:24:03,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/a2438086290d450bb8f059396bfb765e 2024-11-16T04:24:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/055def63138c42ee8dd07fb6f00d1ccc is 50, key is test_row_0/C:col10/1731731043335/Put/seqid=0 2024-11-16T04:24:03,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731103844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741863_1039 (size=12151) 2024-11-16T04:24:03,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731103864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731103864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731103864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731104054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731104074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731104074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T04:24:04,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731104084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731104084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,091 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/055def63138c42ee8dd07fb6f00d1ccc 2024-11-16T04:24:04,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/a48ff5b3ba05449db218e87f1040d899 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/a48ff5b3ba05449db218e87f1040d899 2024-11-16T04:24:04,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/a48ff5b3ba05449db218e87f1040d899, entries=200, sequenceid=137, filesize=14.2 K 2024-11-16T04:24:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/a2438086290d450bb8f059396bfb765e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a2438086290d450bb8f059396bfb765e 2024-11-16T04:24:04,300 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a2438086290d450bb8f059396bfb765e, entries=150, sequenceid=137, filesize=11.9 K 2024-11-16T04:24:04,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/055def63138c42ee8dd07fb6f00d1ccc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/055def63138c42ee8dd07fb6f00d1ccc 2024-11-16T04:24:04,320 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/055def63138c42ee8dd07fb6f00d1ccc, entries=150, sequenceid=137, filesize=11.9 K 2024-11-16T04:24:04,329 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 20bf25e458f1c8ea7c07f170b9f8e62a in 670ms, sequenceid=137, compaction requested=true 2024-11-16T04:24:04,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:04,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T04:24:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T04:24:04,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T04:24:04,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3880 sec 2024-11-16T04:24:04,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4010 sec 2024-11-16T04:24:04,370 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:04,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:24:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:04,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/79ff57693c6c42d7a168b195ee78fd5e is 50, key is test_row_0/A:col10/1731731044367/Put/seqid=0 2024-11-16T04:24:04,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741864_1040 (size=14541) 2024-11-16T04:24:04,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/79ff57693c6c42d7a168b195ee78fd5e 2024-11-16T04:24:04,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731104416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731104420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731104425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731104431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/769c9c63217f446f9862df361ba02453 is 50, key is test_row_0/B:col10/1731731044367/Put/seqid=0 2024-11-16T04:24:04,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741865_1041 (size=12151) 2024-11-16T04:24:04,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/769c9c63217f446f9862df361ba02453 2024-11-16T04:24:04,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/fa5d8d7b1aa4430ebb4abdc6828488d0 is 50, key is test_row_0/C:col10/1731731044367/Put/seqid=0 2024-11-16T04:24:04,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731104536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731104544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731104544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731104545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741866_1042 (size=12151) 2024-11-16T04:24:04,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/fa5d8d7b1aa4430ebb4abdc6828488d0 2024-11-16T04:24:04,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/79ff57693c6c42d7a168b195ee78fd5e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/79ff57693c6c42d7a168b195ee78fd5e 2024-11-16T04:24:04,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/79ff57693c6c42d7a168b195ee78fd5e, entries=200, sequenceid=158, filesize=14.2 K 2024-11-16T04:24:04,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/769c9c63217f446f9862df361ba02453 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/769c9c63217f446f9862df361ba02453 2024-11-16T04:24:04,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/769c9c63217f446f9862df361ba02453, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T04:24:04,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/fa5d8d7b1aa4430ebb4abdc6828488d0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fa5d8d7b1aa4430ebb4abdc6828488d0 2024-11-16T04:24:04,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fa5d8d7b1aa4430ebb4abdc6828488d0, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T04:24:04,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 20bf25e458f1c8ea7c07f170b9f8e62a in 269ms, sequenceid=158, compaction requested=true 2024-11-16T04:24:04,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:04,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:04,640 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:04,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:04,640 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:04,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:04,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:04,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:04,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:04,643 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:04,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:04,643 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:04,643 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:04,643 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/41d58421ca7842c7b4c0969338c6b39a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0262e8f85d7d458a82601ab9d4cee9ea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/a48ff5b3ba05449db218e87f1040d899, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/79ff57693c6c42d7a168b195ee78fd5e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=56.6 K 2024-11-16T04:24:04,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:04,644 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:04,644 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/21a1756515a0488889ec1d7c30c5c6a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1dc198a5b6464449962c8691fd74fa85, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a2438086290d450bb8f059396bfb765e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/769c9c63217f446f9862df361ba02453] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=47.4 K 2024-11-16T04:24:04,645 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 21a1756515a0488889ec1d7c30c5c6a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731042885 2024-11-16T04:24:04,645 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d58421ca7842c7b4c0969338c6b39a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731042885 2024-11-16T04:24:04,645 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dc198a5b6464449962c8691fd74fa85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731731042924 2024-11-16T04:24:04,645 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0262e8f85d7d458a82601ab9d4cee9ea, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731731042924 2024-11-16T04:24:04,646 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a48ff5b3ba05449db218e87f1040d899, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731731043324 2024-11-16T04:24:04,646 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a2438086290d450bb8f059396bfb765e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731731043324 2024-11-16T04:24:04,647 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 769c9c63217f446f9862df361ba02453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731731043726 2024-11-16T04:24:04,647 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ff57693c6c42d7a168b195ee78fd5e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731731043726 2024-11-16T04:24:04,667 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:04,668 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/7ecc512fd6654b4998f20e88c324cc4c is 50, key is test_row_0/A:col10/1731731044367/Put/seqid=0 2024-11-16T04:24:04,677 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:04,678 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/88b5314d7e2a428d8087b2b1ee2dc92d is 50, key is test_row_0/B:col10/1731731044367/Put/seqid=0 2024-11-16T04:24:04,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741867_1043 (size=12493) 2024-11-16T04:24:04,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741868_1044 (size=12493) 2024-11-16T04:24:04,734 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/88b5314d7e2a428d8087b2b1ee2dc92d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/88b5314d7e2a428d8087b2b1ee2dc92d 2024-11-16T04:24:04,738 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/7ecc512fd6654b4998f20e88c324cc4c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/7ecc512fd6654b4998f20e88c324cc4c 2024-11-16T04:24:04,750 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into 7ecc512fd6654b4998f20e88c324cc4c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:04,750 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:04,750 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=12, startTime=1731731044640; duration=0sec 2024-11-16T04:24:04,750 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:04,750 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:04,751 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:04,752 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 88b5314d7e2a428d8087b2b1ee2dc92d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:04,752 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:04,752 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=12, startTime=1731731044640; duration=0sec 2024-11-16T04:24:04,753 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:04,753 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:04,756 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:04,756 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:04,756 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:04,757 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c7757a57dee3427c910da103e1b92337, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/92990378d3ac406aaf6b1aa1d4cf63f5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/055def63138c42ee8dd07fb6f00d1ccc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fa5d8d7b1aa4430ebb4abdc6828488d0] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=47.4 K 2024-11-16T04:24:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:04,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:04,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T04:24:04,758 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7757a57dee3427c910da103e1b92337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731042885 2024-11-16T04:24:04,759 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92990378d3ac406aaf6b1aa1d4cf63f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731731042924 2024-11-16T04:24:04,760 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055def63138c42ee8dd07fb6f00d1ccc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731731043324 2024-11-16T04:24:04,761 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa5d8d7b1aa4430ebb4abdc6828488d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731731043726 2024-11-16T04:24:04,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b68aad53516d4c0a86cd90bd5726d93b is 50, key is test_row_0/A:col10/1731731044424/Put/seqid=0 2024-11-16T04:24:04,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741869_1045 (size=14541) 2024-11-16T04:24:04,802 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:04,803 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/acd7c06ad7994c0bb7c8eeeec5f637fb is 50, key is test_row_0/C:col10/1731731044367/Put/seqid=0 2024-11-16T04:24:04,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741870_1046 (size=12493) 2024-11-16T04:24:04,840 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/acd7c06ad7994c0bb7c8eeeec5f637fb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/acd7c06ad7994c0bb7c8eeeec5f637fb 2024-11-16T04:24:04,854 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into acd7c06ad7994c0bb7c8eeeec5f637fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:04,854 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:04,854 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=12, startTime=1731731044640; duration=0sec 2024-11-16T04:24:04,854 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:04,854 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:04,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731104850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731104854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731104852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731104858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731104973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731104974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731104974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731104974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T04:24:05,094 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:05,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T04:24:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:05,098 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:05,100 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:05,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:05,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731105184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731105184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731105184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731105184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b68aad53516d4c0a86cd90bd5726d93b 2024-11-16T04:24:05,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/d5158021595f4a848eaddb71b4cc6e76 is 50, key is test_row_0/B:col10/1731731044424/Put/seqid=0 2024-11-16T04:24:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:05,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741871_1047 (size=12151) 2024-11-16T04:24:05,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:05,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:05,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731105494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731105495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731105496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731105496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:05,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:05,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/d5158021595f4a848eaddb71b4cc6e76 2024-11-16T04:24:05,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/bcfd6d541d864910b3065ff486e04df6 is 50, key is test_row_0/C:col10/1731731044424/Put/seqid=0 2024-11-16T04:24:05,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741872_1048 (size=12151) 2024-11-16T04:24:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:05,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:05,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731106004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731106014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731106014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731106016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/bcfd6d541d864910b3065ff486e04df6 2024-11-16T04:24:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b68aad53516d4c0a86cd90bd5726d93b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b68aad53516d4c0a86cd90bd5726d93b 2024-11-16T04:24:06,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b68aad53516d4c0a86cd90bd5726d93b, entries=200, sequenceid=176, filesize=14.2 K 2024-11-16T04:24:06,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/d5158021595f4a848eaddb71b4cc6e76 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d5158021595f4a848eaddb71b4cc6e76 2024-11-16T04:24:06,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d5158021595f4a848eaddb71b4cc6e76, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T04:24:06,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/bcfd6d541d864910b3065ff486e04df6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/bcfd6d541d864910b3065ff486e04df6 2024-11-16T04:24:06,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/bcfd6d541d864910b3065ff486e04df6, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T04:24:06,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 20bf25e458f1c8ea7c07f170b9f8e62a in 1349ms, sequenceid=176, compaction requested=false 2024-11-16T04:24:06,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:06,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:06,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:24:06,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/3ff2a69669ca4383ae150d7edbe23fba is 50, key is test_row_0/A:col10/1731731044853/Put/seqid=0 2024-11-16T04:24:06,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741873_1049 (size=12151) 2024-11-16T04:24:06,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/3ff2a69669ca4383ae150d7edbe23fba 2024-11-16T04:24:06,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/1b2505b67c774a23af1127ecd32c5695 is 50, key is test_row_0/B:col10/1731731044853/Put/seqid=0 2024-11-16T04:24:06,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731106193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741874_1050 (size=12151) 2024-11-16T04:24:06,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/1b2505b67c774a23af1127ecd32c5695 2024-11-16T04:24:06,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/0c0350b2818f4c37b5b25c54378bec39 is 50, key is test_row_0/C:col10/1731731044853/Put/seqid=0 2024-11-16T04:24:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:06,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741875_1051 (size=12151) 2024-11-16T04:24:06,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731106313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:06,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:06,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:06,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731106523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:06,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/0c0350b2818f4c37b5b25c54378bec39 2024-11-16T04:24:06,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/3ff2a69669ca4383ae150d7edbe23fba as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3ff2a69669ca4383ae150d7edbe23fba 2024-11-16T04:24:06,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3ff2a69669ca4383ae150d7edbe23fba, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:24:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:06,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/1b2505b67c774a23af1127ecd32c5695 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1b2505b67c774a23af1127ecd32c5695 2024-11-16T04:24:06,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1b2505b67c774a23af1127ecd32c5695, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:24:06,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/0c0350b2818f4c37b5b25c54378bec39 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0c0350b2818f4c37b5b25c54378bec39 2024-11-16T04:24:06,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0c0350b2818f4c37b5b25c54378bec39, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:24:06,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 20bf25e458f1c8ea7c07f170b9f8e62a in 596ms, sequenceid=198, compaction requested=true 2024-11-16T04:24:06,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:06,713 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:06,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:06,715 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:06,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:06,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:06,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:06,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:06,715 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:06,715 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:06,715 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,716 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/7ecc512fd6654b4998f20e88c324cc4c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b68aad53516d4c0a86cd90bd5726d93b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3ff2a69669ca4383ae150d7edbe23fba] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=38.3 K 2024-11-16T04:24:06,716 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ecc512fd6654b4998f20e88c324cc4c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731731043726 2024-11-16T04:24:06,718 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:06,718 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:06,718 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,718 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/88b5314d7e2a428d8087b2b1ee2dc92d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d5158021595f4a848eaddb71b4cc6e76, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1b2505b67c774a23af1127ecd32c5695] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.9 K 2024-11-16T04:24:06,719 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b68aad53516d4c0a86cd90bd5726d93b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731731044415 2024-11-16T04:24:06,719 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b5314d7e2a428d8087b2b1ee2dc92d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731731043726 2024-11-16T04:24:06,720 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ff2a69669ca4383ae150d7edbe23fba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731044796 2024-11-16T04:24:06,720 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d5158021595f4a848eaddb71b4cc6e76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731731044420 2024-11-16T04:24:06,722 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2505b67c774a23af1127ecd32c5695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731044796 2024-11-16T04:24:06,741 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:06,742 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/521f2d85de0740c0afd74efaae5364e4 is 50, key is test_row_0/A:col10/1731731044853/Put/seqid=0 2024-11-16T04:24:06,746 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:06,747 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/cfdbbffb72444962b4924b1c7c985d9b is 50, key is test_row_0/B:col10/1731731044853/Put/seqid=0 2024-11-16T04:24:06,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741876_1052 (size=12595) 2024-11-16T04:24:06,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741877_1053 (size=12595) 2024-11-16T04:24:06,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/521f2d85de0740c0afd74efaae5364e4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/521f2d85de0740c0afd74efaae5364e4 2024-11-16T04:24:06,814 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into 521f2d85de0740c0afd74efaae5364e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:06,814 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:06,814 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731046713; duration=0sec 2024-11-16T04:24:06,815 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:06,815 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:06,815 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:06,818 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:06,818 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:06,818 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,818 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/acd7c06ad7994c0bb7c8eeeec5f637fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/bcfd6d541d864910b3065ff486e04df6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0c0350b2818f4c37b5b25c54378bec39] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=35.9 K 2024-11-16T04:24:06,819 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting acd7c06ad7994c0bb7c8eeeec5f637fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731731043726 2024-11-16T04:24:06,820 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcfd6d541d864910b3065ff486e04df6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731731044420 2024-11-16T04:24:06,821 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c0350b2818f4c37b5b25c54378bec39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731044796 2024-11-16T04:24:06,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T04:24:06,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:06,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:24:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:06,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:06,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/ea28e3705ba24ed9b5f1a2275df2ada4 is 50, key is test_row_0/A:col10/1731731046184/Put/seqid=0 2024-11-16T04:24:06,854 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:06,855 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/7677c31113354f188d24e1764158d808 is 50, key is test_row_0/C:col10/1731731044853/Put/seqid=0 2024-11-16T04:24:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741878_1054 (size=12595) 2024-11-16T04:24:06,876 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/7677c31113354f188d24e1764158d808 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7677c31113354f188d24e1764158d808 2024-11-16T04:24:06,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741879_1055 (size=12151) 2024-11-16T04:24:06,887 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/ea28e3705ba24ed9b5f1a2275df2ada4 2024-11-16T04:24:06,892 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into 7677c31113354f188d24e1764158d808(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:06,892 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:06,892 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731046715; duration=0sec 2024-11-16T04:24:06,892 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:06,892 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:06,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9cd4f34a10cd4515acad4b1d0bf2fd01 is 50, key is test_row_0/B:col10/1731731046184/Put/seqid=0 2024-11-16T04:24:06,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741880_1056 (size=12151) 2024-11-16T04:24:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731106965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731107024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731107024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,031 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,032 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731107033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731107034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,036 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:07,036 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731107073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,202 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/cfdbbffb72444962b4924b1c7c985d9b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cfdbbffb72444962b4924b1c7c985d9b 2024-11-16T04:24:07,228 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into cfdbbffb72444962b4924b1c7c985d9b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:07,228 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:07,229 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731046714; duration=0sec 2024-11-16T04:24:07,229 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:07,229 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:07,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731107283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,319 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9cd4f34a10cd4515acad4b1d0bf2fd01 2024-11-16T04:24:07,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c0be59f6fdd54d78ae879c22bab71166 is 50, key is test_row_0/C:col10/1731731046184/Put/seqid=0 2024-11-16T04:24:07,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741881_1057 (size=12151) 2024-11-16T04:24:07,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c0be59f6fdd54d78ae879c22bab71166 2024-11-16T04:24:07,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/ea28e3705ba24ed9b5f1a2275df2ada4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ea28e3705ba24ed9b5f1a2275df2ada4 2024-11-16T04:24:07,449 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ea28e3705ba24ed9b5f1a2275df2ada4, entries=150, sequenceid=214, filesize=11.9 K 2024-11-16T04:24:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9cd4f34a10cd4515acad4b1d0bf2fd01 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9cd4f34a10cd4515acad4b1d0bf2fd01 2024-11-16T04:24:07,489 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9cd4f34a10cd4515acad4b1d0bf2fd01, entries=150, sequenceid=214, filesize=11.9 K 2024-11-16T04:24:07,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/c0be59f6fdd54d78ae879c22bab71166 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c0be59f6fdd54d78ae879c22bab71166 2024-11-16T04:24:07,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c0be59f6fdd54d78ae879c22bab71166, entries=150, sequenceid=214, filesize=11.9 K 2024-11-16T04:24:07,502 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 20bf25e458f1c8ea7c07f170b9f8e62a in 672ms, sequenceid=214, compaction requested=false 2024-11-16T04:24:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:07,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T04:24:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T04:24:07,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T04:24:07,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4040 sec 2024-11-16T04:24:07,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4190 sec 2024-11-16T04:24:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:07,596 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:07,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T04:24:07,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/803945e86da347e79bf70c2a44af9191 is 50, key is test_row_0/A:col10/1731731047595/Put/seqid=0 2024-11-16T04:24:07,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741882_1058 (size=14541) 2024-11-16T04:24:07,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/803945e86da347e79bf70c2a44af9191 2024-11-16T04:24:07,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/7db7efd0a9a144b487b87b44d98c91f1 is 50, key is test_row_0/B:col10/1731731047595/Put/seqid=0 2024-11-16T04:24:07,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741883_1059 (size=12151) 2024-11-16T04:24:07,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731107686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:07,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:07,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731107793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:07,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731108003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:08,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/7db7efd0a9a144b487b87b44d98c91f1 2024-11-16T04:24:08,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/1b69a00a75ba464182e50e83e1d05d1e is 50, key is test_row_0/C:col10/1731731047595/Put/seqid=0 2024-11-16T04:24:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741884_1060 (size=12151) 2024-11-16T04:24:08,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/1b69a00a75ba464182e50e83e1d05d1e 2024-11-16T04:24:08,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/803945e86da347e79bf70c2a44af9191 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/803945e86da347e79bf70c2a44af9191 2024-11-16T04:24:08,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/803945e86da347e79bf70c2a44af9191, entries=200, sequenceid=238, filesize=14.2 K 2024-11-16T04:24:08,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/7db7efd0a9a144b487b87b44d98c91f1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7db7efd0a9a144b487b87b44d98c91f1 2024-11-16T04:24:08,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7db7efd0a9a144b487b87b44d98c91f1, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T04:24:08,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/1b69a00a75ba464182e50e83e1d05d1e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1b69a00a75ba464182e50e83e1d05d1e 2024-11-16T04:24:08,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1b69a00a75ba464182e50e83e1d05d1e, entries=150, sequenceid=238, filesize=11.9 K 2024-11-16T04:24:08,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 20bf25e458f1c8ea7c07f170b9f8e62a in 627ms, sequenceid=238, compaction requested=true 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:08,224 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:08,224 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:08,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:08,225 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:08,226 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:08,226 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:08,226 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:08,226 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:08,226 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:08,226 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/521f2d85de0740c0afd74efaae5364e4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ea28e3705ba24ed9b5f1a2275df2ada4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/803945e86da347e79bf70c2a44af9191] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=38.4 K 2024-11-16T04:24:08,226 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cfdbbffb72444962b4924b1c7c985d9b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9cd4f34a10cd4515acad4b1d0bf2fd01, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7db7efd0a9a144b487b87b44d98c91f1] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=36.0 K 2024-11-16T04:24:08,227 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 521f2d85de0740c0afd74efaae5364e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731044796 2024-11-16T04:24:08,227 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting cfdbbffb72444962b4924b1c7c985d9b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731044796 2024-11-16T04:24:08,227 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cd4f34a10cd4515acad4b1d0bf2fd01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731731046166 2024-11-16T04:24:08,227 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea28e3705ba24ed9b5f1a2275df2ada4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731731046166 2024-11-16T04:24:08,228 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7db7efd0a9a144b487b87b44d98c91f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731731046955 2024-11-16T04:24:08,228 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803945e86da347e79bf70c2a44af9191, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731731046936 2024-11-16T04:24:08,251 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:08,252 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/c98f299cc2fe4ca4b359c0561471bafb is 50, key is test_row_0/A:col10/1731731047595/Put/seqid=0 2024-11-16T04:24:08,258 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:08,259 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9c74a0ab5158406a8da73777be972f45 is 50, key is test_row_0/B:col10/1731731047595/Put/seqid=0 2024-11-16T04:24:08,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741886_1062 (size=12697) 2024-11-16T04:24:08,316 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:08,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:24:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741885_1061 (size=12697) 2024-11-16T04:24:08,329 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/c98f299cc2fe4ca4b359c0561471bafb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c98f299cc2fe4ca4b359c0561471bafb 2024-11-16T04:24:08,336 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/9c74a0ab5158406a8da73777be972f45 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9c74a0ab5158406a8da73777be972f45 2024-11-16T04:24:08,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/3831483ff0184240acea5d01f63986e3 is 50, key is test_row_0/A:col10/1731731047670/Put/seqid=0 2024-11-16T04:24:08,348 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into c98f299cc2fe4ca4b359c0561471bafb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:08,348 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:08,348 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731048224; duration=0sec 2024-11-16T04:24:08,348 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:08,348 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:08,348 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:08,350 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:08,351 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:08,351 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:08,351 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 9c74a0ab5158406a8da73777be972f45(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:08,351 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:08,351 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7677c31113354f188d24e1764158d808, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c0be59f6fdd54d78ae879c22bab71166, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1b69a00a75ba464182e50e83e1d05d1e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=36.0 K 2024-11-16T04:24:08,351 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731048224; duration=0sec 2024-11-16T04:24:08,351 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:08,351 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:08,352 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7677c31113354f188d24e1764158d808, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731044796 2024-11-16T04:24:08,353 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0be59f6fdd54d78ae879c22bab71166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731731046166 2024-11-16T04:24:08,353 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b69a00a75ba464182e50e83e1d05d1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731731046955 2024-11-16T04:24:08,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741887_1063 (size=14541) 2024-11-16T04:24:08,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/3831483ff0184240acea5d01f63986e3 2024-11-16T04:24:08,371 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:08,372 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/742eff98031c44c3807f597797001a57 is 50, key is test_row_0/C:col10/1731731047595/Put/seqid=0 2024-11-16T04:24:08,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/99c131127b80418b8cffa86be62ca2f2 is 50, key is test_row_0/B:col10/1731731047670/Put/seqid=0 2024-11-16T04:24:08,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741888_1064 (size=12151) 2024-11-16T04:24:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741889_1065 (size=12697) 2024-11-16T04:24:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731108442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:08,454 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/742eff98031c44c3807f597797001a57 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/742eff98031c44c3807f597797001a57 2024-11-16T04:24:08,465 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into 742eff98031c44c3807f597797001a57(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:08,466 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:08,466 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731048224; duration=0sec 2024-11-16T04:24:08,466 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:08,466 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731108553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:08,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731108764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:08,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:08,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:08,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/99c131127b80418b8cffa86be62ca2f2 2024-11-16T04:24:08,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/20e1044ae4d34a6d85adbc62a1be1ec8 is 50, key is test_row_0/C:col10/1731731047670/Put/seqid=0 2024-11-16T04:24:08,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741890_1066 (size=12151) 2024-11-16T04:24:09,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731109033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,035 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731109043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,046 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731109054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,059 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731109063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,067 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731109073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/20e1044ae4d34a6d85adbc62a1be1ec8 2024-11-16T04:24:09,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T04:24:09,264 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:09,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:09,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/3831483ff0184240acea5d01f63986e3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3831483ff0184240acea5d01f63986e3 2024-11-16T04:24:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T04:24:09,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T04:24:09,273 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:09,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3831483ff0184240acea5d01f63986e3, entries=200, sequenceid=253, filesize=14.2 K 2024-11-16T04:24:09,276 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:09,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:09,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/99c131127b80418b8cffa86be62ca2f2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99c131127b80418b8cffa86be62ca2f2 2024-11-16T04:24:09,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99c131127b80418b8cffa86be62ca2f2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T04:24:09,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/20e1044ae4d34a6d85adbc62a1be1ec8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20e1044ae4d34a6d85adbc62a1be1ec8 2024-11-16T04:24:09,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20e1044ae4d34a6d85adbc62a1be1ec8, entries=150, sequenceid=253, filesize=11.9 K 2024-11-16T04:24:09,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 20bf25e458f1c8ea7c07f170b9f8e62a in 980ms, sequenceid=253, compaction requested=false 2024-11-16T04:24:09,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T04:24:09,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T04:24:09,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:09,436 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:24:09,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/1ab76c9a189e46f3ad472a0e99472156 is 50, key is test_row_0/A:col10/1731731048429/Put/seqid=0 2024-11-16T04:24:09,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741891_1067 (size=12301) 2024-11-16T04:24:09,448 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/1ab76c9a189e46f3ad472a0e99472156 2024-11-16T04:24:09,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/235a7afa2aff44aaa0032b32b9eeb715 is 50, key is test_row_0/B:col10/1731731048429/Put/seqid=0 2024-11-16T04:24:09,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741892_1068 (size=12301) 2024-11-16T04:24:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T04:24:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:09,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:09,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731109636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731109743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:09,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/235a7afa2aff44aaa0032b32b9eeb715 2024-11-16T04:24:09,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/0000e37ae6814a8aa4d9e9f981a7a112 is 50, key is test_row_0/C:col10/1731731048429/Put/seqid=0 2024-11-16T04:24:09,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741893_1069 (size=12301) 2024-11-16T04:24:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T04:24:09,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:09,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731109953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:10,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731110264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:10,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:10,280 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/0000e37ae6814a8aa4d9e9f981a7a112 2024-11-16T04:24:10,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/1ab76c9a189e46f3ad472a0e99472156 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1ab76c9a189e46f3ad472a0e99472156 2024-11-16T04:24:10,292 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1ab76c9a189e46f3ad472a0e99472156, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T04:24:10,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/235a7afa2aff44aaa0032b32b9eeb715 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/235a7afa2aff44aaa0032b32b9eeb715 2024-11-16T04:24:10,299 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/235a7afa2aff44aaa0032b32b9eeb715, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T04:24:10,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/0000e37ae6814a8aa4d9e9f981a7a112 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0000e37ae6814a8aa4d9e9f981a7a112 2024-11-16T04:24:10,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0000e37ae6814a8aa4d9e9f981a7a112, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T04:24:10,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 20bf25e458f1c8ea7c07f170b9f8e62a in 873ms, sequenceid=277, compaction requested=true 2024-11-16T04:24:10,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:10,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:10,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T04:24:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T04:24:10,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T04:24:10,313 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-16T04:24:10,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0510 sec 2024-11-16T04:24:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T04:24:10,403 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:10,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T04:24:10,407 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T04:24:10,408 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:10,408 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T04:24:10,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T04:24:10,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:10,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:24:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/fd6b7becb193446fb47c2e5966fa357e is 50, key is test_row_0/A:col10/1731731049626/Put/seqid=0 2024-11-16T04:24:10,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741894_1070 (size=12301) 2024-11-16T04:24:10,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/fd6b7becb193446fb47c2e5966fa357e 2024-11-16T04:24:10,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/bfb5e91e518a4373b4627f5684943437 is 50, key is test_row_0/B:col10/1731731049626/Put/seqid=0 2024-11-16T04:24:10,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741895_1071 (size=12301) 2024-11-16T04:24:10,624 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/bfb5e91e518a4373b4627f5684943437 2024-11-16T04:24:10,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/8f31e487a79240169c2f588eceb3bf72 is 50, key is test_row_0/C:col10/1731731049626/Put/seqid=0 2024-11-16T04:24:10,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741896_1072 (size=12301) 2024-11-16T04:24:10,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/8f31e487a79240169c2f588eceb3bf72 2024-11-16T04:24:10,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/fd6b7becb193446fb47c2e5966fa357e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/fd6b7becb193446fb47c2e5966fa357e 2024-11-16T04:24:10,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/fd6b7becb193446fb47c2e5966fa357e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-16T04:24:10,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/bfb5e91e518a4373b4627f5684943437 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/bfb5e91e518a4373b4627f5684943437 2024-11-16T04:24:10,685 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/bfb5e91e518a4373b4627f5684943437, entries=150, sequenceid=292, filesize=12.0 K 2024-11-16T04:24:10,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/8f31e487a79240169c2f588eceb3bf72 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/8f31e487a79240169c2f588eceb3bf72 2024-11-16T04:24:10,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/8f31e487a79240169c2f588eceb3bf72, entries=150, sequenceid=292, filesize=12.0 K 2024-11-16T04:24:10,701 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 20bf25e458f1c8ea7c07f170b9f8e62a in 139ms, sequenceid=292, compaction requested=true 2024-11-16T04:24:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:10,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T04:24:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T04:24:10,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T04:24:10,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 295 msec 2024-11-16T04:24:10,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 308 msec 2024-11-16T04:24:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T04:24:10,724 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:10,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T04:24:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T04:24:10,727 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:10,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:10,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:10,798 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:10,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:24:10,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/c8ec77bd6165497797190d0ea923b5df is 50, key is test_row_0/A:col10/1731731050777/Put/seqid=0 2024-11-16T04:24:10,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741897_1073 (size=14741) 2024-11-16T04:24:10,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/c8ec77bd6165497797190d0ea923b5df 2024-11-16T04:24:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T04:24:10,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/78e871a59baf4eb3bf877e98fd943c1b is 50, key is test_row_0/B:col10/1731731050777/Put/seqid=0 2024-11-16T04:24:10,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:10,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:10,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:10,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:10,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741898_1074 (size=12301) 2024-11-16T04:24:10,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731110933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:10,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:11,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:11,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T04:24:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731111044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:11,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:11,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731111253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:11,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:11,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/78e871a59baf4eb3bf877e98fd943c1b 2024-11-16T04:24:11,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/910b20f4ce0e4e8db40368e358694bcc is 50, key is test_row_0/C:col10/1731731050777/Put/seqid=0 2024-11-16T04:24:11,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741899_1075 (size=12301) 2024-11-16T04:24:11,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T04:24:11,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:11,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731111563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:11,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:11,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/910b20f4ce0e4e8db40368e358694bcc 2024-11-16T04:24:11,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/c8ec77bd6165497797190d0ea923b5df as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c8ec77bd6165497797190d0ea923b5df 2024-11-16T04:24:11,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c8ec77bd6165497797190d0ea923b5df, entries=200, sequenceid=303, filesize=14.4 K 2024-11-16T04:24:11,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/78e871a59baf4eb3bf877e98fd943c1b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/78e871a59baf4eb3bf877e98fd943c1b 2024-11-16T04:24:11,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/78e871a59baf4eb3bf877e98fd943c1b, entries=150, sequenceid=303, filesize=12.0 K 2024-11-16T04:24:11,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/910b20f4ce0e4e8db40368e358694bcc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/910b20f4ce0e4e8db40368e358694bcc 2024-11-16T04:24:11,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/910b20f4ce0e4e8db40368e358694bcc, entries=150, sequenceid=303, filesize=12.0 K 2024-11-16T04:24:11,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 20bf25e458f1c8ea7c07f170b9f8e62a in 939ms, sequenceid=303, compaction requested=true 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:11,737 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:11,737 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:11,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:11,739 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T04:24:11,739 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:11,739 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,739 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9c74a0ab5158406a8da73777be972f45, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99c131127b80418b8cffa86be62ca2f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/235a7afa2aff44aaa0032b32b9eeb715, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/bfb5e91e518a4373b4627f5684943437, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/78e871a59baf4eb3bf877e98fd943c1b] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=60.3 K 2024-11-16T04:24:11,739 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66581 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T04:24:11,739 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c74a0ab5158406a8da73777be972f45, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731731046955 2024-11-16T04:24:11,739 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:11,740 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,740 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c98f299cc2fe4ca4b359c0561471bafb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3831483ff0184240acea5d01f63986e3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1ab76c9a189e46f3ad472a0e99472156, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/fd6b7becb193446fb47c2e5966fa357e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c8ec77bd6165497797190d0ea923b5df] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=65.0 K 2024-11-16T04:24:11,740 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c131127b80418b8cffa86be62ca2f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731731047670 2024-11-16T04:24:11,740 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98f299cc2fe4ca4b359c0561471bafb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731731046955 2024-11-16T04:24:11,740 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 235a7afa2aff44aaa0032b32b9eeb715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731731048414 2024-11-16T04:24:11,740 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3831483ff0184240acea5d01f63986e3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731731047626 2024-11-16T04:24:11,741 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb5e91e518a4373b4627f5684943437, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731731049623 2024-11-16T04:24:11,741 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ab76c9a189e46f3ad472a0e99472156, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731731048414 2024-11-16T04:24:11,741 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e871a59baf4eb3bf877e98fd943c1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731050777 2024-11-16T04:24:11,741 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd6b7becb193446fb47c2e5966fa357e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731731049623 2024-11-16T04:24:11,741 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8ec77bd6165497797190d0ea923b5df, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731050777 2024-11-16T04:24:11,758 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:11,758 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/8ccb177ecebd4f0da8e34557f573aa50 is 50, key is test_row_0/A:col10/1731731050777/Put/seqid=0 2024-11-16T04:24:11,771 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:11,772 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8ab294215173471583ab6fd116669ff8 is 50, key is test_row_0/B:col10/1731731050777/Put/seqid=0 2024-11-16T04:24:11,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741900_1076 (size=13017) 2024-11-16T04:24:11,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741901_1077 (size=13017) 2024-11-16T04:24:11,790 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8ab294215173471583ab6fd116669ff8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ab294215173471583ab6fd116669ff8 2024-11-16T04:24:11,798 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 8ab294215173471583ab6fd116669ff8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:11,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:11,798 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=11, startTime=1731731051737; duration=0sec 2024-11-16T04:24:11,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:11,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:11,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T04:24:11,802 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61751 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T04:24:11,802 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:11,802 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,802 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/742eff98031c44c3807f597797001a57, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20e1044ae4d34a6d85adbc62a1be1ec8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0000e37ae6814a8aa4d9e9f981a7a112, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/8f31e487a79240169c2f588eceb3bf72, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/910b20f4ce0e4e8db40368e358694bcc] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=60.3 K 2024-11-16T04:24:11,803 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 742eff98031c44c3807f597797001a57, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731731046955 2024-11-16T04:24:11,803 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 20e1044ae4d34a6d85adbc62a1be1ec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731731047670 2024-11-16T04:24:11,804 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0000e37ae6814a8aa4d9e9f981a7a112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731731048414 2024-11-16T04:24:11,804 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f31e487a79240169c2f588eceb3bf72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731731049623 2024-11-16T04:24:11,805 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 910b20f4ce0e4e8db40368e358694bcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731050777 2024-11-16T04:24:11,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T04:24:11,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:11,808 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:24:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b is 50, key is test_row_0/A:col10/1731731050880/Put/seqid=0 2024-11-16T04:24:11,827 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#66 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:11,828 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b3c7c139a01b47f3a6f81c917665fa71 is 50, key is test_row_0/C:col10/1731731050777/Put/seqid=0 2024-11-16T04:24:11,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741902_1078 (size=12301) 2024-11-16T04:24:11,847 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b 2024-11-16T04:24:11,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741903_1079 (size=13017) 2024-11-16T04:24:11,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T04:24:11,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/99ecfc01aa604dfe84c9c490e6898745 is 50, key is test_row_0/B:col10/1731731050880/Put/seqid=0 2024-11-16T04:24:11,875 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b3c7c139a01b47f3a6f81c917665fa71 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b3c7c139a01b47f3a6f81c917665fa71 2024-11-16T04:24:11,886 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into b3c7c139a01b47f3a6f81c917665fa71(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:11,886 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:11,886 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=11, startTime=1731731051737; duration=0sec 2024-11-16T04:24:11,886 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:11,886 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:11,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741904_1080 (size=12301) 2024-11-16T04:24:11,916 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/99ecfc01aa604dfe84c9c490e6898745 2024-11-16T04:24:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/035a03851bac4803ba5c22b37618f0b1 is 50, key is test_row_0/C:col10/1731731050880/Put/seqid=0 2024-11-16T04:24:11,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741905_1081 (size=12301) 2024-11-16T04:24:11,967 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/035a03851bac4803ba5c22b37618f0b1 2024-11-16T04:24:11,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b 2024-11-16T04:24:11,986 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T04:24:11,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/99ecfc01aa604dfe84c9c490e6898745 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99ecfc01aa604dfe84c9c490e6898745 2024-11-16T04:24:12,005 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99ecfc01aa604dfe84c9c490e6898745, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T04:24:12,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/035a03851bac4803ba5c22b37618f0b1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/035a03851bac4803ba5c22b37618f0b1 2024-11-16T04:24:12,017 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/035a03851bac4803ba5c22b37618f0b1, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T04:24:12,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 20bf25e458f1c8ea7c07f170b9f8e62a in 210ms, sequenceid=329, compaction requested=false 2024-11-16T04:24:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:12,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T04:24:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T04:24:12,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T04:24:12,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2900 sec 2024-11-16T04:24:12,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3010 sec 2024-11-16T04:24:12,119 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:12,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:24:12,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b8d7a35c700f47e9896d3c1624eb9147 is 50, key is test_row_0/A:col10/1731731052097/Put/seqid=0 2024-11-16T04:24:12,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741906_1082 (size=14741) 2024-11-16T04:24:12,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b8d7a35c700f47e9896d3c1624eb9147 2024-11-16T04:24:12,191 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/8ccb177ecebd4f0da8e34557f573aa50 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8ccb177ecebd4f0da8e34557f573aa50 2024-11-16T04:24:12,209 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into 8ccb177ecebd4f0da8e34557f573aa50(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:12,209 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:12,210 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=11, startTime=1731731051737; duration=0sec 2024-11-16T04:24:12,210 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:12,210 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:12,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/5cd6d5499eca4481af62e2c8dc190e07 is 50, key is test_row_0/B:col10/1731731052097/Put/seqid=0 2024-11-16T04:24:12,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741907_1083 (size=12301) 2024-11-16T04:24:12,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/5cd6d5499eca4481af62e2c8dc190e07 2024-11-16T04:24:12,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/fb7e3f344b9742c4ba64e391bbcee940 is 50, key is test_row_0/C:col10/1731731052097/Put/seqid=0 2024-11-16T04:24:12,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741908_1084 (size=12301) 2024-11-16T04:24:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731112318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:12,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731112433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:12,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731112644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:12,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/fb7e3f344b9742c4ba64e391bbcee940 2024-11-16T04:24:12,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b8d7a35c700f47e9896d3c1624eb9147 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b8d7a35c700f47e9896d3c1624eb9147 2024-11-16T04:24:12,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b8d7a35c700f47e9896d3c1624eb9147, entries=200, sequenceid=341, filesize=14.4 K 2024-11-16T04:24:12,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/5cd6d5499eca4481af62e2c8dc190e07 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5cd6d5499eca4481af62e2c8dc190e07 2024-11-16T04:24:12,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5cd6d5499eca4481af62e2c8dc190e07, entries=150, sequenceid=341, filesize=12.0 K 2024-11-16T04:24:12,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/fb7e3f344b9742c4ba64e391bbcee940 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fb7e3f344b9742c4ba64e391bbcee940 2024-11-16T04:24:12,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fb7e3f344b9742c4ba64e391bbcee940, entries=150, sequenceid=341, filesize=12.0 K 2024-11-16T04:24:12,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 20bf25e458f1c8ea7c07f170b9f8e62a in 623ms, sequenceid=341, compaction requested=true 2024-11-16T04:24:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:12,742 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:12,742 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:12,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:12,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:12,743 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:12,743 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:12,744 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:12,744 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:12,744 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:12,744 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:12,744 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8ccb177ecebd4f0da8e34557f573aa50, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b8d7a35c700f47e9896d3c1624eb9147] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=39.1 K 2024-11-16T04:24:12,744 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ab294215173471583ab6fd116669ff8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99ecfc01aa604dfe84c9c490e6898745, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5cd6d5499eca4481af62e2c8dc190e07] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=36.7 K 2024-11-16T04:24:12,744 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ab294215173471583ab6fd116669ff8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731050777 2024-11-16T04:24:12,744 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ccb177ecebd4f0da8e34557f573aa50, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731050777 2024-11-16T04:24:12,745 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 99ecfc01aa604dfe84c9c490e6898745, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731050880 2024-11-16T04:24:12,745 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d4dfaa3fdd24d699f60c0b85dbbfd7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731050880 2024-11-16T04:24:12,745 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd6d5499eca4481af62e2c8dc190e07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731731052092 2024-11-16T04:24:12,746 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8d7a35c700f47e9896d3c1624eb9147, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731731052077 2024-11-16T04:24:12,755 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:12,756 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:12,756 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/d46f7b6ad6454f1dbea5353f06e70ee9 is 50, key is test_row_0/A:col10/1731731052097/Put/seqid=0 2024-11-16T04:24:12,757 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/b378bb0889f44f25aff808f885daf40c is 50, key is test_row_0/B:col10/1731731052097/Put/seqid=0 2024-11-16T04:24:12,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741910_1086 (size=13119) 2024-11-16T04:24:12,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741909_1085 (size=13119) 2024-11-16T04:24:12,803 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/b378bb0889f44f25aff808f885daf40c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b378bb0889f44f25aff808f885daf40c 2024-11-16T04:24:12,810 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into b378bb0889f44f25aff808f885daf40c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:12,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:12,811 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731052742; duration=0sec 2024-11-16T04:24:12,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:12,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:12,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:12,814 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:12,814 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:12,814 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:12,814 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b3c7c139a01b47f3a6f81c917665fa71, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/035a03851bac4803ba5c22b37618f0b1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fb7e3f344b9742c4ba64e391bbcee940] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=36.7 K 2024-11-16T04:24:12,814 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c7c139a01b47f3a6f81c917665fa71, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731050777 2024-11-16T04:24:12,815 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 035a03851bac4803ba5c22b37618f0b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731050880 2024-11-16T04:24:12,815 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7e3f344b9742c4ba64e391bbcee940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731731052092 2024-11-16T04:24:12,826 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:12,827 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/69bcca11d6c9471abcbc1024950db097 is 50, key is test_row_0/C:col10/1731731052097/Put/seqid=0 2024-11-16T04:24:12,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741911_1087 (size=13119) 2024-11-16T04:24:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T04:24:12,874 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:12,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:12,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T04:24:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T04:24:12,878 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:12,879 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:12,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:12,957 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:12,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T04:24:12,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/bf1afd0a92eb44759a78c9056a7337f2 is 50, key is test_row_0/A:col10/1731731052316/Put/seqid=0 2024-11-16T04:24:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T04:24:12,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731112995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:12,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741912_1088 (size=14741) 2024-11-16T04:24:13,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:13,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50594 deadline: 1731731113074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,075 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50572 deadline: 1731731113073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,080 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50606 deadline: 1731731113084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,087 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50614 deadline: 1731731113104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731113104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,106 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 20bf25e458f1c8ea7c07f170b9f8e62a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T04:24:13,208 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/d46f7b6ad6454f1dbea5353f06e70ee9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d46f7b6ad6454f1dbea5353f06e70ee9 2024-11-16T04:24:13,216 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into d46f7b6ad6454f1dbea5353f06e70ee9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:13,216 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:13,216 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731052742; duration=0sec 2024-11-16T04:24:13,216 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:13,216 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:13,242 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/69bcca11d6c9471abcbc1024950db097 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/69bcca11d6c9471abcbc1024950db097 2024-11-16T04:24:13,247 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into 69bcca11d6c9471abcbc1024950db097(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:13,247 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:13,247 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731052742; duration=0sec 2024-11-16T04:24:13,248 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:13,248 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:13,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731113314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/bf1afd0a92eb44759a78c9056a7337f2 2024-11-16T04:24:13,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/ace15fefb57e457bbd73fc48693ebe66 is 50, key is test_row_0/B:col10/1731731052316/Put/seqid=0 2024-11-16T04:24:13,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741913_1089 (size=12301) 2024-11-16T04:24:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T04:24:13,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:13,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731113623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:13,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:13,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/ace15fefb57e457bbd73fc48693ebe66 2024-11-16T04:24:13,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:13,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/1af5b5783c9243128661e63856a5baa4 is 50, key is test_row_0/C:col10/1731731052316/Put/seqid=0 2024-11-16T04:24:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741914_1090 (size=12301) 2024-11-16T04:24:13,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/1af5b5783c9243128661e63856a5baa4 2024-11-16T04:24:13,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/bf1afd0a92eb44759a78c9056a7337f2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bf1afd0a92eb44759a78c9056a7337f2 2024-11-16T04:24:13,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bf1afd0a92eb44759a78c9056a7337f2, entries=200, sequenceid=369, filesize=14.4 K 2024-11-16T04:24:13,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/ace15fefb57e457bbd73fc48693ebe66 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ace15fefb57e457bbd73fc48693ebe66 2024-11-16T04:24:13,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ace15fefb57e457bbd73fc48693ebe66, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T04:24:13,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/1af5b5783c9243128661e63856a5baa4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1af5b5783c9243128661e63856a5baa4 2024-11-16T04:24:13,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1af5b5783c9243128661e63856a5baa4, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T04:24:13,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 20bf25e458f1c8ea7c07f170b9f8e62a in 914ms, sequenceid=369, compaction requested=false 2024-11-16T04:24:13,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:13,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T04:24:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:13,972 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T04:24:13,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/ba79a8ffde28440bac45a615145c0ec7 is 50, key is test_row_0/A:col10/1731731052993/Put/seqid=0 2024-11-16T04:24:13,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741915_1091 (size=12301) 2024-11-16T04:24:13,991 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/ba79a8ffde28440bac45a615145c0ec7 2024-11-16T04:24:14,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T04:24:14,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/642d31f01e534d109fec89d92e7dca3c is 50, key is test_row_0/B:col10/1731731052993/Put/seqid=0 2024-11-16T04:24:14,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741916_1092 (size=12301) 2024-11-16T04:24:14,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:14,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:14,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:14,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731114256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:14,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731114363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:14,432 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/642d31f01e534d109fec89d92e7dca3c 2024-11-16T04:24:14,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/259376ffd8774668983ed7e645258739 is 50, key is test_row_0/C:col10/1731731052993/Put/seqid=0 2024-11-16T04:24:14,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741917_1093 (size=12301) 2024-11-16T04:24:14,449 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/259376ffd8774668983ed7e645258739 2024-11-16T04:24:14,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/ba79a8ffde28440bac45a615145c0ec7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ba79a8ffde28440bac45a615145c0ec7 2024-11-16T04:24:14,465 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ba79a8ffde28440bac45a615145c0ec7, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T04:24:14,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/642d31f01e534d109fec89d92e7dca3c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/642d31f01e534d109fec89d92e7dca3c 2024-11-16T04:24:14,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/642d31f01e534d109fec89d92e7dca3c, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T04:24:14,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/259376ffd8774668983ed7e645258739 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/259376ffd8774668983ed7e645258739 2024-11-16T04:24:14,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/259376ffd8774668983ed7e645258739, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T04:24:14,487 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 20bf25e458f1c8ea7c07f170b9f8e62a in 515ms, sequenceid=381, compaction requested=true 2024-11-16T04:24:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:14,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T04:24:14,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T04:24:14,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T04:24:14,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6090 sec 2024-11-16T04:24:14,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6180 sec 2024-11-16T04:24:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:14,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-16T04:24:14,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/bdd9e21f39df499882209262b92b6d62 is 50, key is test_row_0/A:col10/1731731054254/Put/seqid=0 2024-11-16T04:24:14,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741918_1094 (size=14741) 2024-11-16T04:24:14,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/bdd9e21f39df499882209262b92b6d62 2024-11-16T04:24:14,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:14,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731114604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:14,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/19c0038b89fa42f693cfebfcf73d257f is 50, key is test_row_0/B:col10/1731731054254/Put/seqid=0 2024-11-16T04:24:14,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741919_1095 (size=12301) 2024-11-16T04:24:14,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/19c0038b89fa42f693cfebfcf73d257f 2024-11-16T04:24:14,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/3c888a874d2e44549a92f90febac5eb5 is 50, key is test_row_0/C:col10/1731731054254/Put/seqid=0 2024-11-16T04:24:14,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741920_1096 (size=12301) 2024-11-16T04:24:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731114713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:14,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731114924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:14,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T04:24:15,024 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:15,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T04:24:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T04:24:15,027 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:15,028 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:15,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:15,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/3c888a874d2e44549a92f90febac5eb5 2024-11-16T04:24:15,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/bdd9e21f39df499882209262b92b6d62 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bdd9e21f39df499882209262b92b6d62 2024-11-16T04:24:15,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bdd9e21f39df499882209262b92b6d62, entries=200, sequenceid=408, filesize=14.4 K 2024-11-16T04:24:15,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/19c0038b89fa42f693cfebfcf73d257f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/19c0038b89fa42f693cfebfcf73d257f 2024-11-16T04:24:15,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/19c0038b89fa42f693cfebfcf73d257f, entries=150, sequenceid=408, filesize=12.0 K 2024-11-16T04:24:15,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/3c888a874d2e44549a92f90febac5eb5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/3c888a874d2e44549a92f90febac5eb5 2024-11-16T04:24:15,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/3c888a874d2e44549a92f90febac5eb5, entries=150, sequenceid=408, filesize=12.0 K 2024-11-16T04:24:15,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 20bf25e458f1c8ea7c07f170b9f8e62a in 488ms, sequenceid=408, compaction requested=true 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:15,066 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:15,066 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:15,068 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:15,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54902 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:15,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:15,068 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:15,068 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,068 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,068 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b378bb0889f44f25aff808f885daf40c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ace15fefb57e457bbd73fc48693ebe66, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/642d31f01e534d109fec89d92e7dca3c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/19c0038b89fa42f693cfebfcf73d257f] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=48.8 K 2024-11-16T04:24:15,068 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d46f7b6ad6454f1dbea5353f06e70ee9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bf1afd0a92eb44759a78c9056a7337f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ba79a8ffde28440bac45a615145c0ec7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bdd9e21f39df499882209262b92b6d62] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=53.6 K 2024-11-16T04:24:15,069 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b378bb0889f44f25aff808f885daf40c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731731052092 2024-11-16T04:24:15,069 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d46f7b6ad6454f1dbea5353f06e70ee9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731731052092 2024-11-16T04:24:15,069 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ace15fefb57e457bbd73fc48693ebe66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731731052290 2024-11-16T04:24:15,069 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf1afd0a92eb44759a78c9056a7337f2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731731052286 2024-11-16T04:24:15,069 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 642d31f01e534d109fec89d92e7dca3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731731052987 2024-11-16T04:24:15,070 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba79a8ffde28440bac45a615145c0ec7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731731052987 2024-11-16T04:24:15,070 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 19c0038b89fa42f693cfebfcf73d257f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731731054241 2024-11-16T04:24:15,070 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdd9e21f39df499882209262b92b6d62, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731731054241 2024-11-16T04:24:15,082 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:15,083 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/16c875a0cfc94d3da8a614745699017e is 50, key is test_row_0/B:col10/1731731054254/Put/seqid=0 2024-11-16T04:24:15,091 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:15,091 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/89718d8b320c44d2be9944003e69af84 is 50, key is test_row_0/A:col10/1731731054254/Put/seqid=0 2024-11-16T04:24:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741922_1098 (size=13255) 2024-11-16T04:24:15,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741921_1097 (size=13255) 2024-11-16T04:24:15,112 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/16c875a0cfc94d3da8a614745699017e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/16c875a0cfc94d3da8a614745699017e 2024-11-16T04:24:15,119 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 16c875a0cfc94d3da8a614745699017e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:15,119 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:15,119 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=12, startTime=1731731055066; duration=0sec 2024-11-16T04:24:15,119 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:15,119 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:15,119 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:15,122 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:15,122 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:15,122 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,123 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/69bcca11d6c9471abcbc1024950db097, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1af5b5783c9243128661e63856a5baa4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/259376ffd8774668983ed7e645258739, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/3c888a874d2e44549a92f90febac5eb5] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=48.8 K 2024-11-16T04:24:15,124 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 69bcca11d6c9471abcbc1024950db097, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731731052092 2024-11-16T04:24:15,125 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af5b5783c9243128661e63856a5baa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731731052290 2024-11-16T04:24:15,126 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 259376ffd8774668983ed7e645258739, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731731052987 2024-11-16T04:24:15,126 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c888a874d2e44549a92f90febac5eb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731731054241 2024-11-16T04:24:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T04:24:15,140 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#86 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:15,141 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b2863afb9fee411899c3b9d2d3e8a47b is 50, key is test_row_0/C:col10/1731731054254/Put/seqid=0 2024-11-16T04:24:15,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741923_1099 (size=13255) 2024-11-16T04:24:15,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T04:24:15,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T04:24:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/6cabcd404afe4e5fb9a788b1a6260a83 is 50, key is test_row_0/A:col10/1731731054598/Put/seqid=0 2024-11-16T04:24:15,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741924_1100 (size=9857) 2024-11-16T04:24:15,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/6cabcd404afe4e5fb9a788b1a6260a83 2024-11-16T04:24:15,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/cd1fbe3e77d9400280681161ad15bdfc is 50, key is test_row_0/B:col10/1731731054598/Put/seqid=0 2024-11-16T04:24:15,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741925_1101 (size=9857) 2024-11-16T04:24:15,240 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/cd1fbe3e77d9400280681161ad15bdfc 2024-11-16T04:24:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:15,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:15,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/7453339c109142adb1fd06b8e68d0037 is 50, key is test_row_0/C:col10/1731731054598/Put/seqid=0 2024-11-16T04:24:15,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741926_1102 (size=9857) 2024-11-16T04:24:15,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/7453339c109142adb1fd06b8e68d0037 2024-11-16T04:24:15,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/6cabcd404afe4e5fb9a788b1a6260a83 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6cabcd404afe4e5fb9a788b1a6260a83 2024-11-16T04:24:15,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6cabcd404afe4e5fb9a788b1a6260a83, entries=100, sequenceid=418, filesize=9.6 K 2024-11-16T04:24:15,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/cd1fbe3e77d9400280681161ad15bdfc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cd1fbe3e77d9400280681161ad15bdfc 2024-11-16T04:24:15,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cd1fbe3e77d9400280681161ad15bdfc, entries=100, sequenceid=418, filesize=9.6 K 2024-11-16T04:24:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/7453339c109142adb1fd06b8e68d0037 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7453339c109142adb1fd06b8e68d0037 2024-11-16T04:24:15,309 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7453339c109142adb1fd06b8e68d0037, entries=100, sequenceid=418, filesize=9.6 K 2024-11-16T04:24:15,310 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=100.63 KB/103050 for 20bf25e458f1c8ea7c07f170b9f8e62a in 126ms, sequenceid=418, compaction requested=false 2024-11-16T04:24:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T04:24:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T04:24:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:15,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:15,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T04:24:15,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T04:24:15,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 284 msec 2024-11-16T04:24:15,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 297 msec 2024-11-16T04:24:15,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/9dd934da482b4584b8a179c817a0d56e is 50, key is test_row_0/A:col10/1731731055302/Put/seqid=0 2024-11-16T04:24:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T04:24:15,344 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:15,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T04:24:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:15,347 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:15,348 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:15,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:15,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741927_1103 (size=14741) 2024-11-16T04:24:15,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731115413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:15,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:15,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:15,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,514 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/89718d8b320c44d2be9944003e69af84 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/89718d8b320c44d2be9944003e69af84 2024-11-16T04:24:15,525 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into 89718d8b320c44d2be9944003e69af84(size=12.9 K), total size for store is 22.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:15,525 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:15,525 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=12, startTime=1731731055066; duration=0sec 2024-11-16T04:24:15,525 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:15,525 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:15,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731115524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:15,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:15,555 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b2863afb9fee411899c3b9d2d3e8a47b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b2863afb9fee411899c3b9d2d3e8a47b 2024-11-16T04:24:15,563 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into b2863afb9fee411899c3b9d2d3e8a47b(size=12.9 K), total size for store is 22.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:15,563 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:15,563 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=12, startTime=1731731055066; duration=0sec 2024-11-16T04:24:15,563 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:15,563 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:15,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:15,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:15,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731115743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:15,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/9dd934da482b4584b8a179c817a0d56e 2024-11-16T04:24:15,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/a930c1d73f96458d9cab20971137d9cf is 50, key is test_row_0/B:col10/1731731055302/Put/seqid=0 2024-11-16T04:24:15,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741928_1104 (size=12301) 2024-11-16T04:24:15,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:15,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:15,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:16,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731116054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:16,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:16,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/a930c1d73f96458d9cab20971137d9cf 2024-11-16T04:24:16,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/20a8ced41dc04c2fafd7a07cedf18410 is 50, key is test_row_0/C:col10/1731731055302/Put/seqid=0 2024-11-16T04:24:16,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741929_1105 (size=12301) 2024-11-16T04:24:16,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:16,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731116563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:16,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:16,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:16,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/20a8ced41dc04c2fafd7a07cedf18410 2024-11-16T04:24:16,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/9dd934da482b4584b8a179c817a0d56e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9dd934da482b4584b8a179c817a0d56e 2024-11-16T04:24:16,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9dd934da482b4584b8a179c817a0d56e, entries=200, sequenceid=437, filesize=14.4 K 2024-11-16T04:24:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/a930c1d73f96458d9cab20971137d9cf as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a930c1d73f96458d9cab20971137d9cf 2024-11-16T04:24:16,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a930c1d73f96458d9cab20971137d9cf, entries=150, sequenceid=437, filesize=12.0 K 2024-11-16T04:24:16,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/20a8ced41dc04c2fafd7a07cedf18410 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20a8ced41dc04c2fafd7a07cedf18410 2024-11-16T04:24:16,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20a8ced41dc04c2fafd7a07cedf18410, entries=150, sequenceid=437, filesize=12.0 K 2024-11-16T04:24:16,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 20bf25e458f1c8ea7c07f170b9f8e62a in 1344ms, sequenceid=437, compaction requested=true 2024-11-16T04:24:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:16,659 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:16,659 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:16,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:16,661 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37853 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:16,661 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:16,661 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,661 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/89718d8b320c44d2be9944003e69af84, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6cabcd404afe4e5fb9a788b1a6260a83, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9dd934da482b4584b8a179c817a0d56e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=37.0 K 2024-11-16T04:24:16,663 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89718d8b320c44d2be9944003e69af84, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731731054241 2024-11-16T04:24:16,663 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:16,663 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:16,663 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,663 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/16c875a0cfc94d3da8a614745699017e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cd1fbe3e77d9400280681161ad15bdfc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a930c1d73f96458d9cab20971137d9cf] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=34.6 K 2024-11-16T04:24:16,664 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c875a0cfc94d3da8a614745699017e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731731054241 2024-11-16T04:24:16,664 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cabcd404afe4e5fb9a788b1a6260a83, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731731054598 2024-11-16T04:24:16,665 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd934da482b4584b8a179c817a0d56e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731731055283 2024-11-16T04:24:16,665 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1fbe3e77d9400280681161ad15bdfc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731731054598 2024-11-16T04:24:16,666 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a930c1d73f96458d9cab20971137d9cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731731055283 2024-11-16T04:24:16,688 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:16,689 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/d0dd58ef54ba43908e004a8bd329f2e9 is 50, key is test_row_0/A:col10/1731731055302/Put/seqid=0 2024-11-16T04:24:16,696 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:16,697 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/7ca44023ac174e96bbf47571da34302f is 50, key is test_row_0/B:col10/1731731055302/Put/seqid=0 2024-11-16T04:24:16,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741930_1106 (size=13357) 2024-11-16T04:24:16,717 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/d0dd58ef54ba43908e004a8bd329f2e9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d0dd58ef54ba43908e004a8bd329f2e9 2024-11-16T04:24:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741931_1107 (size=13357) 2024-11-16T04:24:16,729 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into d0dd58ef54ba43908e004a8bd329f2e9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:16,729 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:16,729 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731056659; duration=0sec 2024-11-16T04:24:16,729 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/7ca44023ac174e96bbf47571da34302f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7ca44023ac174e96bbf47571da34302f 2024-11-16T04:24:16,729 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:16,729 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:16,729 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:16,731 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:16,731 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:16,731 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,731 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b2863afb9fee411899c3b9d2d3e8a47b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7453339c109142adb1fd06b8e68d0037, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20a8ced41dc04c2fafd7a07cedf18410] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=34.6 K 2024-11-16T04:24:16,732 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2863afb9fee411899c3b9d2d3e8a47b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731731054241 2024-11-16T04:24:16,738 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7453339c109142adb1fd06b8e68d0037, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1731731054598 2024-11-16T04:24:16,739 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20a8ced41dc04c2fafd7a07cedf18410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731731055283 2024-11-16T04:24:16,742 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into 7ca44023ac174e96bbf47571da34302f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:16,742 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:16,742 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731056659; duration=0sec 2024-11-16T04:24:16,743 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:16,743 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:16,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T04:24:16,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:16,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T04:24:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/be28e12dc2204cc78cde6860d66e8bd4 is 50, key is test_row_0/A:col10/1731731055395/Put/seqid=0 2024-11-16T04:24:16,753 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:16,753 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/d8da6737023b460187acbddfac372ef7 is 50, key is test_row_0/C:col10/1731731055302/Put/seqid=0 2024-11-16T04:24:16,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741932_1108 (size=12301) 2024-11-16T04:24:16,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741933_1109 (size=13357) 2024-11-16T04:24:16,773 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/d8da6737023b460187acbddfac372ef7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/d8da6737023b460187acbddfac372ef7 2024-11-16T04:24:16,815 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into d8da6737023b460187acbddfac372ef7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:16,816 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:16,816 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731056659; duration=0sec 2024-11-16T04:24:16,816 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:16,816 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:17,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/be28e12dc2204cc78cde6860d66e8bd4 2024-11-16T04:24:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/f33e1241be9d400f9e9455cc99ca9b94 is 50, key is test_row_0/B:col10/1731731055395/Put/seqid=0 2024-11-16T04:24:17,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741934_1110 (size=12301) 2024-11-16T04:24:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:17,575 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/f33e1241be9d400f9e9455cc99ca9b94 2024-11-16T04:24:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:17,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:17,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/cde29de0069e42a8adcd21060c9ef89a is 50, key is test_row_0/C:col10/1731731055395/Put/seqid=0 2024-11-16T04:24:17,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741935_1111 (size=12301) 2024-11-16T04:24:17,597 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/cde29de0069e42a8adcd21060c9ef89a 2024-11-16T04:24:17,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/be28e12dc2204cc78cde6860d66e8bd4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/be28e12dc2204cc78cde6860d66e8bd4 2024-11-16T04:24:17,612 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/be28e12dc2204cc78cde6860d66e8bd4, entries=150, sequenceid=458, filesize=12.0 K 2024-11-16T04:24:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/f33e1241be9d400f9e9455cc99ca9b94 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/f33e1241be9d400f9e9455cc99ca9b94 2024-11-16T04:24:17,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/f33e1241be9d400f9e9455cc99ca9b94, entries=150, sequenceid=458, filesize=12.0 K 2024-11-16T04:24:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/cde29de0069e42a8adcd21060c9ef89a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/cde29de0069e42a8adcd21060c9ef89a 2024-11-16T04:24:17,636 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/cde29de0069e42a8adcd21060c9ef89a, entries=150, sequenceid=458, filesize=12.0 K 2024-11-16T04:24:17,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=100.63 KB/103050 for 20bf25e458f1c8ea7c07f170b9f8e62a in 894ms, sequenceid=458, compaction requested=false 2024-11-16T04:24:17,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:17,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:17,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T04:24:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T04:24:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:17,638 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:17,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T04:24:17,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T04:24:17,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b209579cc2da41039ae1b59fb904567e is 50, key is test_row_0/A:col10/1731731057633/Put/seqid=0 2024-11-16T04:24:17,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2920 sec 2024-11-16T04:24:17,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.3040 sec 2024-11-16T04:24:17,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741936_1112 (size=14741) 2024-11-16T04:24:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731117732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:17,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731117844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731118053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:18,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b209579cc2da41039ae1b59fb904567e 2024-11-16T04:24:18,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/2f064bac354540888d7d5c0358b40ffc is 50, key is test_row_0/B:col10/1731731057633/Put/seqid=0 2024-11-16T04:24:18,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741937_1113 (size=12301) 2024-11-16T04:24:18,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:18,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731118364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:18,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/2f064bac354540888d7d5c0358b40ffc 2024-11-16T04:24:18,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/6462097328c642239ca4330d915772ca is 50, key is test_row_0/C:col10/1731731057633/Put/seqid=0 2024-11-16T04:24:18,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741938_1114 (size=12301) 2024-11-16T04:24:18,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/6462097328c642239ca4330d915772ca 2024-11-16T04:24:18,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/b209579cc2da41039ae1b59fb904567e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b209579cc2da41039ae1b59fb904567e 2024-11-16T04:24:18,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b209579cc2da41039ae1b59fb904567e, entries=200, sequenceid=478, filesize=14.4 K 2024-11-16T04:24:18,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/2f064bac354540888d7d5c0358b40ffc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/2f064bac354540888d7d5c0358b40ffc 2024-11-16T04:24:18,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/2f064bac354540888d7d5c0358b40ffc, entries=150, sequenceid=478, filesize=12.0 K 2024-11-16T04:24:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/6462097328c642239ca4330d915772ca as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/6462097328c642239ca4330d915772ca 2024-11-16T04:24:18,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/6462097328c642239ca4330d915772ca, entries=150, sequenceid=478, filesize=12.0 K 2024-11-16T04:24:18,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 20bf25e458f1c8ea7c07f170b9f8e62a in 929ms, sequenceid=478, compaction requested=true 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:18,567 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:18,567 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 20bf25e458f1c8ea7c07f170b9f8e62a:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:18,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:18,568 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:18,568 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:18,568 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/A is initiating minor compaction (all files) 2024-11-16T04:24:18,568 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/B is initiating minor compaction (all files) 2024-11-16T04:24:18,568 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/B in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:18,568 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/A in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:18,569 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d0dd58ef54ba43908e004a8bd329f2e9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/be28e12dc2204cc78cde6860d66e8bd4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b209579cc2da41039ae1b59fb904567e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=39.5 K 2024-11-16T04:24:18,569 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7ca44023ac174e96bbf47571da34302f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/f33e1241be9d400f9e9455cc99ca9b94, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/2f064bac354540888d7d5c0358b40ffc] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=37.1 K 2024-11-16T04:24:18,569 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca44023ac174e96bbf47571da34302f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731731055283 2024-11-16T04:24:18,569 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0dd58ef54ba43908e004a8bd329f2e9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731731055283 2024-11-16T04:24:18,569 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f33e1241be9d400f9e9455cc99ca9b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731731055389 2024-11-16T04:24:18,569 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting be28e12dc2204cc78cde6860d66e8bd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731731055389 2024-11-16T04:24:18,570 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b209579cc2da41039ae1b59fb904567e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731731057620 2024-11-16T04:24:18,570 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f064bac354540888d7d5c0358b40ffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731731057631 2024-11-16T04:24:18,577 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#B#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:18,578 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:18,578 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/e4e6176d93254b7da5ab63ec7eb95eac is 50, key is test_row_0/B:col10/1731731057633/Put/seqid=0 2024-11-16T04:24:18,578 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/f4e51957633447068cb883fb7fc0f1a4 is 50, key is test_row_0/A:col10/1731731057633/Put/seqid=0 2024-11-16T04:24:18,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741939_1115 (size=13459) 2024-11-16T04:24:18,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741940_1116 (size=13459) 2024-11-16T04:24:18,591 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/f4e51957633447068cb883fb7fc0f1a4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/f4e51957633447068cb883fb7fc0f1a4 2024-11-16T04:24:18,598 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/A of 20bf25e458f1c8ea7c07f170b9f8e62a into f4e51957633447068cb883fb7fc0f1a4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:18,598 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:18,598 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/A, priority=13, startTime=1731731058567; duration=0sec 2024-11-16T04:24:18,598 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:18,598 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:A 2024-11-16T04:24:18,598 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:18,599 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:18,599 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 20bf25e458f1c8ea7c07f170b9f8e62a/C is initiating minor compaction (all files) 2024-11-16T04:24:18,599 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 20bf25e458f1c8ea7c07f170b9f8e62a/C in TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:18,599 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/d8da6737023b460187acbddfac372ef7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/cde29de0069e42a8adcd21060c9ef89a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/6462097328c642239ca4330d915772ca] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp, totalSize=37.1 K 2024-11-16T04:24:18,600 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8da6737023b460187acbddfac372ef7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1731731055283 2024-11-16T04:24:18,600 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cde29de0069e42a8adcd21060c9ef89a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731731055389 2024-11-16T04:24:18,600 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6462097328c642239ca4330d915772ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1731731057631 2024-11-16T04:24:18,607 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 20bf25e458f1c8ea7c07f170b9f8e62a#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:18,607 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b4ee374ef4f647cf8c573b613ba65816 is 50, key is test_row_0/C:col10/1731731057633/Put/seqid=0 2024-11-16T04:24:18,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741941_1117 (size=13459) 2024-11-16T04:24:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:18,875 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:18,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T04:24:18,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/8e2955180b02450b8bbba77a6d5e706f is 50, key is test_row_0/A:col10/1731731057711/Put/seqid=0 2024-11-16T04:24:18,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741942_1118 (size=14741) 2024-11-16T04:24:18,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731118917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:18,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:18,994 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/e4e6176d93254b7da5ab63ec7eb95eac as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/e4e6176d93254b7da5ab63ec7eb95eac 2024-11-16T04:24:19,000 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/B of 20bf25e458f1c8ea7c07f170b9f8e62a into e4e6176d93254b7da5ab63ec7eb95eac(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:19,000 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:19,000 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/B, priority=13, startTime=1731731058567; duration=0sec 2024-11-16T04:24:19,000 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:19,000 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:B 2024-11-16T04:24:19,017 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b4ee374ef4f647cf8c573b613ba65816 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b4ee374ef4f647cf8c573b613ba65816 2024-11-16T04:24:19,022 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 20bf25e458f1c8ea7c07f170b9f8e62a/C of 20bf25e458f1c8ea7c07f170b9f8e62a into b4ee374ef4f647cf8c573b613ba65816(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:19,022 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:19,022 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., storeName=20bf25e458f1c8ea7c07f170b9f8e62a/C, priority=13, startTime=1731731058567; duration=0sec 2024-11-16T04:24:19,022 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:19,022 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 20bf25e458f1c8ea7c07f170b9f8e62a:C 2024-11-16T04:24:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731119024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:19,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731119234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:19,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/8e2955180b02450b8bbba77a6d5e706f 2024-11-16T04:24:19,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8923d5b08356496da5299479764729f0 is 50, key is test_row_0/B:col10/1731731057711/Put/seqid=0 2024-11-16T04:24:19,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741943_1119 (size=12301) 2024-11-16T04:24:19,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T04:24:19,514 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:19,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-16T04:24:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T04:24:19,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:19,518 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:19,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:19,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731119543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:19,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T04:24:19,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T04:24:19,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:19,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:19,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:19,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8923d5b08356496da5299479764729f0 2024-11-16T04:24:19,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/a1d15eca20b94a14ad860bcad7fd383d is 50, key is test_row_0/C:col10/1731731057711/Put/seqid=0 2024-11-16T04:24:19,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741944_1120 (size=12301) 2024-11-16T04:24:19,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T04:24:19,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:19,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:19,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:19,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T04:24:19,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T04:24:19,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:19,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:19,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:19,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:19,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:20,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50608 deadline: 1731731120053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:24:20,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/a1d15eca20b94a14ad860bcad7fd383d 2024-11-16T04:24:20,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/8e2955180b02450b8bbba77a6d5e706f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8e2955180b02450b8bbba77a6d5e706f 2024-11-16T04:24:20,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T04:24:20,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:20,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:20,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:20,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:20,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:20,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:20,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8e2955180b02450b8bbba77a6d5e706f, entries=200, sequenceid=497, filesize=14.4 K 2024-11-16T04:24:20,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8923d5b08356496da5299479764729f0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8923d5b08356496da5299479764729f0 2024-11-16T04:24:20,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8923d5b08356496da5299479764729f0, entries=150, sequenceid=497, filesize=12.0 K 2024-11-16T04:24:20,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/a1d15eca20b94a14ad860bcad7fd383d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/a1d15eca20b94a14ad860bcad7fd383d 2024-11-16T04:24:20,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T04:24:20,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/a1d15eca20b94a14ad860bcad7fd383d, entries=150, sequenceid=497, filesize=12.0 K 2024-11-16T04:24:20,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 20bf25e458f1c8ea7c07f170b9f8e62a in 1273ms, sequenceid=497, compaction requested=false 2024-11-16T04:24:20,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:20,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T04:24:20,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:20,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T04:24:20,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/00f09b7a92b44d239398e1609f94fff8 is 50, key is test_row_0/A:col10/1731731058901/Put/seqid=0 2024-11-16T04:24:20,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741945_1121 (size=12301) 2024-11-16T04:24:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T04:24:20,699 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/00f09b7a92b44d239398e1609f94fff8 2024-11-16T04:24:20,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8204e2e50d624c439e66128d41ff1ae0 is 50, key is test_row_0/B:col10/1731731058901/Put/seqid=0 2024-11-16T04:24:20,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741946_1122 (size=12301) 2024-11-16T04:24:20,712 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8204e2e50d624c439e66128d41ff1ae0 2024-11-16T04:24:20,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b09cba7c14694df6ab8e9c4855ed5c08 is 50, key is test_row_0/C:col10/1731731058901/Put/seqid=0 2024-11-16T04:24:20,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741947_1123 (size=12301) 2024-11-16T04:24:20,806 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T04:24:20,806 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:20,807 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T04:24:20,807 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:20,808 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T04:24:20,808 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:20,808 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T04:24:20,808 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:20,811 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:20,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:20,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:20,812 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:20,812 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:21,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. as already flushing 2024-11-16T04:24:21,078 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T04:24:21,078 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:21,078 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:21,078 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:21,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:21,132 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b09cba7c14694df6ab8e9c4855ed5c08 2024-11-16T04:24:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/00f09b7a92b44d239398e1609f94fff8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/00f09b7a92b44d239398e1609f94fff8 2024-11-16T04:24:21,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/00f09b7a92b44d239398e1609f94fff8, entries=150, sequenceid=517, filesize=12.0 K 2024-11-16T04:24:21,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/8204e2e50d624c439e66128d41ff1ae0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8204e2e50d624c439e66128d41ff1ae0 2024-11-16T04:24:21,154 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8204e2e50d624c439e66128d41ff1ae0, entries=150, sequenceid=517, filesize=12.0 K 2024-11-16T04:24:21,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/b09cba7c14694df6ab8e9c4855ed5c08 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b09cba7c14694df6ab8e9c4855ed5c08 2024-11-16T04:24:21,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b09cba7c14694df6ab8e9c4855ed5c08, entries=150, sequenceid=517, filesize=12.0 K 2024-11-16T04:24:21,161 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=6.71 KB/6870 for 20bf25e458f1c8ea7c07f170b9f8e62a in 877ms, sequenceid=517, compaction requested=true 2024-11-16T04:24:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 20bf25e458f1c8ea7c07f170b9f8e62a: 2024-11-16T04:24:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-16T04:24:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-16T04:24:21,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T04:24:21,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6440 sec 2024-11-16T04:24:21,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.6510 sec 2024-11-16T04:24:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T04:24:21,664 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:22,336 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T04:24:23,109 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T04:24:23,109 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:23,109 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,109 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:23,125 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T04:24:23,125 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:23,125 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,125 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:23,186 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T04:24:23,186 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:23,186 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,186 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:23,186 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,215 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T04:24:23,215 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:23,215 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,215 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 261 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3646 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3636 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T04:24:23,216 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1604 2024-11-16T04:24:23,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4804 rows 2024-11-16T04:24:23,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1620 2024-11-16T04:24:23,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4846 rows 2024-11-16T04:24:23,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:24:23,217 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:24:23,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:23,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:23,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T04:24:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T04:24:23,229 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731063229"}]},"ts":"1731731063229"} 2024-11-16T04:24:23,231 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T04:24:23,231 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T04:24:23,234 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:24:23,238 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, UNASSIGN}] 2024-11-16T04:24:23,239 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, UNASSIGN 2024-11-16T04:24:23,241 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=20bf25e458f1c8ea7c07f170b9f8e62a, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:23,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, UNASSIGN because future has completed 2024-11-16T04:24:23,245 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=fda3efb342d8,45455,1731731034087, table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T04:24:23,246 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:24:23,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:24:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T04:24:23,404 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:23,404 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:24:23,405 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 20bf25e458f1c8ea7c07f170b9f8e62a, disabling compactions & flushes 2024-11-16T04:24:23,405 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:23,405 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:23,405 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. after waiting 0 ms 2024-11-16T04:24:23,405 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:23,405 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 20bf25e458f1c8ea7c07f170b9f8e62a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T04:24:23,412 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/9f98c3a149c34c9e99210e495783e893 is 50, key is test_row_0/A:col10/1731731063123/Put/seqid=0 2024-11-16T04:24:23,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741948_1124 (size=9857) 2024-11-16T04:24:23,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T04:24:23,817 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/9f98c3a149c34c9e99210e495783e893 2024-11-16T04:24:23,826 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/171ed20fa436480fa1f45fc522a7eb14 is 50, key is test_row_0/B:col10/1731731063123/Put/seqid=0 2024-11-16T04:24:23,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741949_1125 (size=9857) 2024-11-16T04:24:23,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T04:24:24,234 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/171ed20fa436480fa1f45fc522a7eb14 2024-11-16T04:24:24,247 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/adc10af227624fff88072bdc139e335a is 50, key is test_row_0/C:col10/1731731063123/Put/seqid=0 2024-11-16T04:24:24,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741950_1126 (size=9857) 2024-11-16T04:24:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T04:24:24,652 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/adc10af227624fff88072bdc139e335a 2024-11-16T04:24:24,662 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/A/9f98c3a149c34c9e99210e495783e893 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9f98c3a149c34c9e99210e495783e893 2024-11-16T04:24:24,668 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9f98c3a149c34c9e99210e495783e893, entries=100, sequenceid=525, filesize=9.6 K 2024-11-16T04:24:24,669 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/B/171ed20fa436480fa1f45fc522a7eb14 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/171ed20fa436480fa1f45fc522a7eb14 2024-11-16T04:24:24,677 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/171ed20fa436480fa1f45fc522a7eb14, entries=100, sequenceid=525, filesize=9.6 K 2024-11-16T04:24:24,678 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/.tmp/C/adc10af227624fff88072bdc139e335a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/adc10af227624fff88072bdc139e335a 2024-11-16T04:24:24,686 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/adc10af227624fff88072bdc139e335a, entries=100, sequenceid=525, filesize=9.6 K 2024-11-16T04:24:24,687 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 20bf25e458f1c8ea7c07f170b9f8e62a in 1282ms, sequenceid=525, compaction requested=true 2024-11-16T04:24:24,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6c4c8ab912b24d2c81b9fb5e25319acb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/5f840beffacb45618483a0807fb205f3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/001c0f9ed7a84019a398ee2553b5a4c0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1a1cc6d526e1452789f1e36c43224d85, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/01c67b74f18048d4baadf398c6c0f5fc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/41d58421ca7842c7b4c0969338c6b39a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/4454592c001c42b9b6e09bdb4e03b621, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0262e8f85d7d458a82601ab9d4cee9ea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/a48ff5b3ba05449db218e87f1040d899, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/79ff57693c6c42d7a168b195ee78fd5e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/7ecc512fd6654b4998f20e88c324cc4c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b68aad53516d4c0a86cd90bd5726d93b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/521f2d85de0740c0afd74efaae5364e4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3ff2a69669ca4383ae150d7edbe23fba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ea28e3705ba24ed9b5f1a2275df2ada4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/803945e86da347e79bf70c2a44af9191, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c98f299cc2fe4ca4b359c0561471bafb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3831483ff0184240acea5d01f63986e3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1ab76c9a189e46f3ad472a0e99472156, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/fd6b7becb193446fb47c2e5966fa357e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c8ec77bd6165497797190d0ea923b5df, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8ccb177ecebd4f0da8e34557f573aa50, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b8d7a35c700f47e9896d3c1624eb9147, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d46f7b6ad6454f1dbea5353f06e70ee9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bf1afd0a92eb44759a78c9056a7337f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ba79a8ffde28440bac45a615145c0ec7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bdd9e21f39df499882209262b92b6d62, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/89718d8b320c44d2be9944003e69af84, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6cabcd404afe4e5fb9a788b1a6260a83, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9dd934da482b4584b8a179c817a0d56e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d0dd58ef54ba43908e004a8bd329f2e9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/be28e12dc2204cc78cde6860d66e8bd4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b209579cc2da41039ae1b59fb904567e] to archive 2024-11-16T04:24:24,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:24:24,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6c4c8ab912b24d2c81b9fb5e25319acb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6c4c8ab912b24d2c81b9fb5e25319acb 2024-11-16T04:24:24,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/5f840beffacb45618483a0807fb205f3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/5f840beffacb45618483a0807fb205f3 2024-11-16T04:24:24,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/001c0f9ed7a84019a398ee2553b5a4c0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/001c0f9ed7a84019a398ee2553b5a4c0 2024-11-16T04:24:24,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1a1cc6d526e1452789f1e36c43224d85 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1a1cc6d526e1452789f1e36c43224d85 2024-11-16T04:24:24,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/01c67b74f18048d4baadf398c6c0f5fc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/01c67b74f18048d4baadf398c6c0f5fc 2024-11-16T04:24:24,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/41d58421ca7842c7b4c0969338c6b39a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/41d58421ca7842c7b4c0969338c6b39a 2024-11-16T04:24:24,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/4454592c001c42b9b6e09bdb4e03b621 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/4454592c001c42b9b6e09bdb4e03b621 2024-11-16T04:24:24,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0262e8f85d7d458a82601ab9d4cee9ea to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0262e8f85d7d458a82601ab9d4cee9ea 2024-11-16T04:24:24,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/a48ff5b3ba05449db218e87f1040d899 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/a48ff5b3ba05449db218e87f1040d899 2024-11-16T04:24:24,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/79ff57693c6c42d7a168b195ee78fd5e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/79ff57693c6c42d7a168b195ee78fd5e 2024-11-16T04:24:24,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/7ecc512fd6654b4998f20e88c324cc4c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/7ecc512fd6654b4998f20e88c324cc4c 2024-11-16T04:24:24,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b68aad53516d4c0a86cd90bd5726d93b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b68aad53516d4c0a86cd90bd5726d93b 2024-11-16T04:24:24,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/521f2d85de0740c0afd74efaae5364e4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/521f2d85de0740c0afd74efaae5364e4 2024-11-16T04:24:24,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3ff2a69669ca4383ae150d7edbe23fba to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3ff2a69669ca4383ae150d7edbe23fba 2024-11-16T04:24:24,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ea28e3705ba24ed9b5f1a2275df2ada4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ea28e3705ba24ed9b5f1a2275df2ada4 2024-11-16T04:24:24,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/803945e86da347e79bf70c2a44af9191 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/803945e86da347e79bf70c2a44af9191 2024-11-16T04:24:24,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c98f299cc2fe4ca4b359c0561471bafb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c98f299cc2fe4ca4b359c0561471bafb 2024-11-16T04:24:24,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3831483ff0184240acea5d01f63986e3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/3831483ff0184240acea5d01f63986e3 2024-11-16T04:24:24,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1ab76c9a189e46f3ad472a0e99472156 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/1ab76c9a189e46f3ad472a0e99472156 2024-11-16T04:24:24,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/fd6b7becb193446fb47c2e5966fa357e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/fd6b7becb193446fb47c2e5966fa357e 2024-11-16T04:24:24,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c8ec77bd6165497797190d0ea923b5df to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/c8ec77bd6165497797190d0ea923b5df 2024-11-16T04:24:24,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8ccb177ecebd4f0da8e34557f573aa50 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8ccb177ecebd4f0da8e34557f573aa50 2024-11-16T04:24:24,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/0d4dfaa3fdd24d699f60c0b85dbbfd7b 2024-11-16T04:24:24,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b8d7a35c700f47e9896d3c1624eb9147 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b8d7a35c700f47e9896d3c1624eb9147 2024-11-16T04:24:24,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d46f7b6ad6454f1dbea5353f06e70ee9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d46f7b6ad6454f1dbea5353f06e70ee9 2024-11-16T04:24:24,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bf1afd0a92eb44759a78c9056a7337f2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bf1afd0a92eb44759a78c9056a7337f2 2024-11-16T04:24:24,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ba79a8ffde28440bac45a615145c0ec7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/ba79a8ffde28440bac45a615145c0ec7 2024-11-16T04:24:24,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bdd9e21f39df499882209262b92b6d62 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/bdd9e21f39df499882209262b92b6d62 2024-11-16T04:24:24,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/89718d8b320c44d2be9944003e69af84 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/89718d8b320c44d2be9944003e69af84 2024-11-16T04:24:24,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6cabcd404afe4e5fb9a788b1a6260a83 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/6cabcd404afe4e5fb9a788b1a6260a83 2024-11-16T04:24:24,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9dd934da482b4584b8a179c817a0d56e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9dd934da482b4584b8a179c817a0d56e 2024-11-16T04:24:24,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d0dd58ef54ba43908e004a8bd329f2e9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/d0dd58ef54ba43908e004a8bd329f2e9 2024-11-16T04:24:24,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/be28e12dc2204cc78cde6860d66e8bd4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/be28e12dc2204cc78cde6860d66e8bd4 2024-11-16T04:24:24,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b209579cc2da41039ae1b59fb904567e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/b209579cc2da41039ae1b59fb904567e 2024-11-16T04:24:24,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b97e529fd62a465297dda59615aea769, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ac70378ec22f4d2c839ff5aa45a9f05b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5bd7b7e61ebc419297bf72f58acc7ea1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d934bc5225364538b21ab0f88c1363ba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9283ba3f9e024b67b52cf9b3dfb56a46, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/21a1756515a0488889ec1d7c30c5c6a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ac735a5db324aa2bac25834c221f6f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1dc198a5b6464449962c8691fd74fa85, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a2438086290d450bb8f059396bfb765e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/88b5314d7e2a428d8087b2b1ee2dc92d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/769c9c63217f446f9862df361ba02453, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d5158021595f4a848eaddb71b4cc6e76, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cfdbbffb72444962b4924b1c7c985d9b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1b2505b67c774a23af1127ecd32c5695, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9cd4f34a10cd4515acad4b1d0bf2fd01, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9c74a0ab5158406a8da73777be972f45, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7db7efd0a9a144b487b87b44d98c91f1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99c131127b80418b8cffa86be62ca2f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/235a7afa2aff44aaa0032b32b9eeb715, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/bfb5e91e518a4373b4627f5684943437, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ab294215173471583ab6fd116669ff8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/78e871a59baf4eb3bf877e98fd943c1b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99ecfc01aa604dfe84c9c490e6898745, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b378bb0889f44f25aff808f885daf40c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5cd6d5499eca4481af62e2c8dc190e07, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ace15fefb57e457bbd73fc48693ebe66, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/642d31f01e534d109fec89d92e7dca3c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/16c875a0cfc94d3da8a614745699017e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/19c0038b89fa42f693cfebfcf73d257f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cd1fbe3e77d9400280681161ad15bdfc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7ca44023ac174e96bbf47571da34302f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a930c1d73f96458d9cab20971137d9cf, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/f33e1241be9d400f9e9455cc99ca9b94, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/2f064bac354540888d7d5c0358b40ffc] to archive 2024-11-16T04:24:24,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:24:24,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b97e529fd62a465297dda59615aea769 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b97e529fd62a465297dda59615aea769 2024-11-16T04:24:24,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ac70378ec22f4d2c839ff5aa45a9f05b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ac70378ec22f4d2c839ff5aa45a9f05b 2024-11-16T04:24:24,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5bd7b7e61ebc419297bf72f58acc7ea1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5bd7b7e61ebc419297bf72f58acc7ea1 2024-11-16T04:24:24,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d934bc5225364538b21ab0f88c1363ba to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d934bc5225364538b21ab0f88c1363ba 2024-11-16T04:24:24,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9283ba3f9e024b67b52cf9b3dfb56a46 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9283ba3f9e024b67b52cf9b3dfb56a46 2024-11-16T04:24:24,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/21a1756515a0488889ec1d7c30c5c6a0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/21a1756515a0488889ec1d7c30c5c6a0 2024-11-16T04:24:24,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ac735a5db324aa2bac25834c221f6f2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ac735a5db324aa2bac25834c221f6f2 2024-11-16T04:24:24,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1dc198a5b6464449962c8691fd74fa85 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1dc198a5b6464449962c8691fd74fa85 2024-11-16T04:24:24,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a2438086290d450bb8f059396bfb765e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a2438086290d450bb8f059396bfb765e 2024-11-16T04:24:24,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/88b5314d7e2a428d8087b2b1ee2dc92d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/88b5314d7e2a428d8087b2b1ee2dc92d 2024-11-16T04:24:24,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/769c9c63217f446f9862df361ba02453 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/769c9c63217f446f9862df361ba02453 2024-11-16T04:24:24,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d5158021595f4a848eaddb71b4cc6e76 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/d5158021595f4a848eaddb71b4cc6e76 2024-11-16T04:24:24,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cfdbbffb72444962b4924b1c7c985d9b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cfdbbffb72444962b4924b1c7c985d9b 2024-11-16T04:24:24,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1b2505b67c774a23af1127ecd32c5695 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/1b2505b67c774a23af1127ecd32c5695 2024-11-16T04:24:24,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9cd4f34a10cd4515acad4b1d0bf2fd01 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9cd4f34a10cd4515acad4b1d0bf2fd01 2024-11-16T04:24:24,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9c74a0ab5158406a8da73777be972f45 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/9c74a0ab5158406a8da73777be972f45 2024-11-16T04:24:24,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7db7efd0a9a144b487b87b44d98c91f1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7db7efd0a9a144b487b87b44d98c91f1 2024-11-16T04:24:24,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99c131127b80418b8cffa86be62ca2f2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99c131127b80418b8cffa86be62ca2f2 2024-11-16T04:24:24,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/235a7afa2aff44aaa0032b32b9eeb715 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/235a7afa2aff44aaa0032b32b9eeb715 2024-11-16T04:24:24,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/bfb5e91e518a4373b4627f5684943437 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/bfb5e91e518a4373b4627f5684943437 2024-11-16T04:24:24,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ab294215173471583ab6fd116669ff8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8ab294215173471583ab6fd116669ff8 2024-11-16T04:24:24,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/78e871a59baf4eb3bf877e98fd943c1b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/78e871a59baf4eb3bf877e98fd943c1b 2024-11-16T04:24:24,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99ecfc01aa604dfe84c9c490e6898745 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/99ecfc01aa604dfe84c9c490e6898745 2024-11-16T04:24:24,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b378bb0889f44f25aff808f885daf40c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/b378bb0889f44f25aff808f885daf40c 2024-11-16T04:24:24,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5cd6d5499eca4481af62e2c8dc190e07 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/5cd6d5499eca4481af62e2c8dc190e07 2024-11-16T04:24:24,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ace15fefb57e457bbd73fc48693ebe66 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/ace15fefb57e457bbd73fc48693ebe66 2024-11-16T04:24:24,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/642d31f01e534d109fec89d92e7dca3c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/642d31f01e534d109fec89d92e7dca3c 2024-11-16T04:24:24,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/16c875a0cfc94d3da8a614745699017e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/16c875a0cfc94d3da8a614745699017e 2024-11-16T04:24:24,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/19c0038b89fa42f693cfebfcf73d257f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/19c0038b89fa42f693cfebfcf73d257f 2024-11-16T04:24:24,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cd1fbe3e77d9400280681161ad15bdfc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/cd1fbe3e77d9400280681161ad15bdfc 2024-11-16T04:24:24,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7ca44023ac174e96bbf47571da34302f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/7ca44023ac174e96bbf47571da34302f 2024-11-16T04:24:24,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a930c1d73f96458d9cab20971137d9cf to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/a930c1d73f96458d9cab20971137d9cf 2024-11-16T04:24:24,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/f33e1241be9d400f9e9455cc99ca9b94 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/f33e1241be9d400f9e9455cc99ca9b94 2024-11-16T04:24:24,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/2f064bac354540888d7d5c0358b40ffc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/2f064bac354540888d7d5c0358b40ffc 2024-11-16T04:24:24,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/093ca87cca7e426f9d560c298a9b146d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c04f841a361c43e9b866d24e9dc01328, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c599bfb980ea4912bd4b7d09ad532fe7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/44a9b3e4042440f094b8133805d6f172, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/ec2c5ddbf6914123942a4ca6d196c72b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c7757a57dee3427c910da103e1b92337, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/49f1e4f39464419580bfcffe29508d88, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/92990378d3ac406aaf6b1aa1d4cf63f5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/055def63138c42ee8dd07fb6f00d1ccc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/acd7c06ad7994c0bb7c8eeeec5f637fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fa5d8d7b1aa4430ebb4abdc6828488d0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/bcfd6d541d864910b3065ff486e04df6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7677c31113354f188d24e1764158d808, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0c0350b2818f4c37b5b25c54378bec39, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c0be59f6fdd54d78ae879c22bab71166, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/742eff98031c44c3807f597797001a57, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1b69a00a75ba464182e50e83e1d05d1e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20e1044ae4d34a6d85adbc62a1be1ec8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0000e37ae6814a8aa4d9e9f981a7a112, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/8f31e487a79240169c2f588eceb3bf72, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b3c7c139a01b47f3a6f81c917665fa71, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/910b20f4ce0e4e8db40368e358694bcc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/035a03851bac4803ba5c22b37618f0b1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/69bcca11d6c9471abcbc1024950db097, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fb7e3f344b9742c4ba64e391bbcee940, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1af5b5783c9243128661e63856a5baa4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/259376ffd8774668983ed7e645258739, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b2863afb9fee411899c3b9d2d3e8a47b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/3c888a874d2e44549a92f90febac5eb5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7453339c109142adb1fd06b8e68d0037, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/d8da6737023b460187acbddfac372ef7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20a8ced41dc04c2fafd7a07cedf18410, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/cde29de0069e42a8adcd21060c9ef89a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/6462097328c642239ca4330d915772ca] to archive 2024-11-16T04:24:24,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:24:24,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/093ca87cca7e426f9d560c298a9b146d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/093ca87cca7e426f9d560c298a9b146d 2024-11-16T04:24:24,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c04f841a361c43e9b866d24e9dc01328 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c04f841a361c43e9b866d24e9dc01328 2024-11-16T04:24:24,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c599bfb980ea4912bd4b7d09ad532fe7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c599bfb980ea4912bd4b7d09ad532fe7 2024-11-16T04:24:24,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/44a9b3e4042440f094b8133805d6f172 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/44a9b3e4042440f094b8133805d6f172 2024-11-16T04:24:24,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/ec2c5ddbf6914123942a4ca6d196c72b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/ec2c5ddbf6914123942a4ca6d196c72b 2024-11-16T04:24:24,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c7757a57dee3427c910da103e1b92337 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c7757a57dee3427c910da103e1b92337 2024-11-16T04:24:24,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/49f1e4f39464419580bfcffe29508d88 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/49f1e4f39464419580bfcffe29508d88 2024-11-16T04:24:24,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/92990378d3ac406aaf6b1aa1d4cf63f5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/92990378d3ac406aaf6b1aa1d4cf63f5 2024-11-16T04:24:24,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/055def63138c42ee8dd07fb6f00d1ccc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/055def63138c42ee8dd07fb6f00d1ccc 2024-11-16T04:24:24,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/acd7c06ad7994c0bb7c8eeeec5f637fb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/acd7c06ad7994c0bb7c8eeeec5f637fb 2024-11-16T04:24:24,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fa5d8d7b1aa4430ebb4abdc6828488d0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fa5d8d7b1aa4430ebb4abdc6828488d0 2024-11-16T04:24:24,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/bcfd6d541d864910b3065ff486e04df6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/bcfd6d541d864910b3065ff486e04df6 2024-11-16T04:24:24,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7677c31113354f188d24e1764158d808 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7677c31113354f188d24e1764158d808 2024-11-16T04:24:24,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0c0350b2818f4c37b5b25c54378bec39 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0c0350b2818f4c37b5b25c54378bec39 2024-11-16T04:24:24,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c0be59f6fdd54d78ae879c22bab71166 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/c0be59f6fdd54d78ae879c22bab71166 2024-11-16T04:24:24,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/742eff98031c44c3807f597797001a57 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/742eff98031c44c3807f597797001a57 2024-11-16T04:24:24,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1b69a00a75ba464182e50e83e1d05d1e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1b69a00a75ba464182e50e83e1d05d1e 2024-11-16T04:24:24,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20e1044ae4d34a6d85adbc62a1be1ec8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20e1044ae4d34a6d85adbc62a1be1ec8 2024-11-16T04:24:24,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0000e37ae6814a8aa4d9e9f981a7a112 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/0000e37ae6814a8aa4d9e9f981a7a112 2024-11-16T04:24:24,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/8f31e487a79240169c2f588eceb3bf72 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/8f31e487a79240169c2f588eceb3bf72 2024-11-16T04:24:24,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b3c7c139a01b47f3a6f81c917665fa71 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b3c7c139a01b47f3a6f81c917665fa71 2024-11-16T04:24:24,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/910b20f4ce0e4e8db40368e358694bcc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/910b20f4ce0e4e8db40368e358694bcc 2024-11-16T04:24:24,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/035a03851bac4803ba5c22b37618f0b1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/035a03851bac4803ba5c22b37618f0b1 2024-11-16T04:24:24,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/69bcca11d6c9471abcbc1024950db097 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/69bcca11d6c9471abcbc1024950db097 2024-11-16T04:24:24,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fb7e3f344b9742c4ba64e391bbcee940 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/fb7e3f344b9742c4ba64e391bbcee940 2024-11-16T04:24:24,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1af5b5783c9243128661e63856a5baa4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/1af5b5783c9243128661e63856a5baa4 2024-11-16T04:24:24,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/259376ffd8774668983ed7e645258739 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/259376ffd8774668983ed7e645258739 2024-11-16T04:24:24,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b2863afb9fee411899c3b9d2d3e8a47b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b2863afb9fee411899c3b9d2d3e8a47b 2024-11-16T04:24:24,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/3c888a874d2e44549a92f90febac5eb5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/3c888a874d2e44549a92f90febac5eb5 2024-11-16T04:24:24,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7453339c109142adb1fd06b8e68d0037 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/7453339c109142adb1fd06b8e68d0037 2024-11-16T04:24:24,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/d8da6737023b460187acbddfac372ef7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/d8da6737023b460187acbddfac372ef7 2024-11-16T04:24:24,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20a8ced41dc04c2fafd7a07cedf18410 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/20a8ced41dc04c2fafd7a07cedf18410 2024-11-16T04:24:24,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/cde29de0069e42a8adcd21060c9ef89a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/cde29de0069e42a8adcd21060c9ef89a 2024-11-16T04:24:24,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/6462097328c642239ca4330d915772ca to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/6462097328c642239ca4330d915772ca 2024-11-16T04:24:24,852 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/recovered.edits/528.seqid, newMaxSeqId=528, maxSeqId=1 2024-11-16T04:24:24,853 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. 2024-11-16T04:24:24,853 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 20bf25e458f1c8ea7c07f170b9f8e62a: Waiting for close lock at 1731731063405Running coprocessor pre-close hooks at 1731731063405Disabling compacts and flushes for region at 1731731063405Disabling writes for close at 1731731063405Obtaining lock to block concurrent updates at 1731731063405Preparing flush snapshotting stores in 20bf25e458f1c8ea7c07f170b9f8e62a at 1731731063405Finished memstore snapshotting TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731731063406 (+1 ms)Flushing stores of TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a. at 1731731063406Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/A: creating writer at 1731731063406Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/A: appending metadata at 1731731063411 (+5 ms)Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/A: closing flushed file at 1731731063411Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/B: creating writer at 1731731063822 (+411 ms)Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/B: appending metadata at 1731731063826 (+4 ms)Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/B: closing flushed file at 1731731063826Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/C: creating writer at 1731731064244 (+418 ms)Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/C: appending metadata at 1731731064246 (+2 ms)Flushing 20bf25e458f1c8ea7c07f170b9f8e62a/C: closing flushed file at 1731731064246Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f886dd7: reopening flushed file at 1731731064661 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ea642e: reopening flushed file at 1731731064668 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c6191ae: reopening flushed file at 1731731064677 (+9 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 20bf25e458f1c8ea7c07f170b9f8e62a in 1282ms, sequenceid=525, compaction requested=true at 1731731064687 (+10 ms)Writing region close event to WAL at 1731731064847 (+160 ms)Running coprocessor post-close hooks at 1731731064853 (+6 ms)Closed at 1731731064853 2024-11-16T04:24:24,857 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:24,857 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=20bf25e458f1c8ea7c07f170b9f8e62a, regionState=CLOSED 2024-11-16T04:24:24,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:24:24,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-16T04:24:24,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 20bf25e458f1c8ea7c07f170b9f8e62a, server=fda3efb342d8,45455,1731731034087 in 1.6150 sec 2024-11-16T04:24:24,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-16T04:24:24,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=20bf25e458f1c8ea7c07f170b9f8e62a, UNASSIGN in 1.6260 sec 2024-11-16T04:24:24,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T04:24:24,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6340 sec 2024-11-16T04:24:24,872 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731064872"}]},"ts":"1731731064872"} 2024-11-16T04:24:24,874 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T04:24:24,874 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T04:24:24,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6520 sec 2024-11-16T04:24:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T04:24:25,374 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:25,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T04:24:25,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,383 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T04:24:25,385 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,390 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:25,395 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/recovered.edits] 2024-11-16T04:24:25,399 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/00f09b7a92b44d239398e1609f94fff8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/00f09b7a92b44d239398e1609f94fff8 2024-11-16T04:24:25,400 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8e2955180b02450b8bbba77a6d5e706f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/8e2955180b02450b8bbba77a6d5e706f 2024-11-16T04:24:25,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9f98c3a149c34c9e99210e495783e893 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/9f98c3a149c34c9e99210e495783e893 2024-11-16T04:24:25,403 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/f4e51957633447068cb883fb7fc0f1a4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/A/f4e51957633447068cb883fb7fc0f1a4 2024-11-16T04:24:25,407 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/171ed20fa436480fa1f45fc522a7eb14 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/171ed20fa436480fa1f45fc522a7eb14 2024-11-16T04:24:25,409 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8204e2e50d624c439e66128d41ff1ae0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8204e2e50d624c439e66128d41ff1ae0 2024-11-16T04:24:25,410 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8923d5b08356496da5299479764729f0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/8923d5b08356496da5299479764729f0 2024-11-16T04:24:25,411 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/e4e6176d93254b7da5ab63ec7eb95eac to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/B/e4e6176d93254b7da5ab63ec7eb95eac 2024-11-16T04:24:25,414 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/a1d15eca20b94a14ad860bcad7fd383d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/a1d15eca20b94a14ad860bcad7fd383d 2024-11-16T04:24:25,415 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/adc10af227624fff88072bdc139e335a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/adc10af227624fff88072bdc139e335a 2024-11-16T04:24:25,417 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b09cba7c14694df6ab8e9c4855ed5c08 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b09cba7c14694df6ab8e9c4855ed5c08 2024-11-16T04:24:25,418 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b4ee374ef4f647cf8c573b613ba65816 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/C/b4ee374ef4f647cf8c573b613ba65816 2024-11-16T04:24:25,422 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/recovered.edits/528.seqid to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a/recovered.edits/528.seqid 2024-11-16T04:24:25,422 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/20bf25e458f1c8ea7c07f170b9f8e62a 2024-11-16T04:24:25,422 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T04:24:25,428 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T04:24:25,437 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T04:24:25,470 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T04:24:25,472 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,472 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T04:24:25,472 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731731065472"}]},"ts":"9223372036854775807"} 2024-11-16T04:24:25,476 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T04:24:25,476 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 20bf25e458f1c8ea7c07f170b9f8e62a, NAME => 'TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T04:24:25,476 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T04:24:25,476 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731731065476"}]},"ts":"9223372036854775807"} 2024-11-16T04:24:25,479 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T04:24:25,480 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-16T04:24:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T04:24:25,494 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T04:24:25,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:25,504 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1846551405_22 at /127.0.0.1:49618 [Waiting for operation #286] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1374812648_22 at /127.0.0.1:49474 [Waiting for operation #300] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fda3efb342d8:45455-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 344) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8467 (was 9331) 2024-11-16T04:24:25,512 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=8466 2024-11-16T04:24:25,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:24:25,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:24:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:25,516 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T04:24:25,516 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:25,516 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-16T04:24:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T04:24:25,517 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T04:24:25,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741951_1127 (size=959) 2024-11-16T04:24:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T04:24:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T04:24:25,925 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:24:25,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741952_1128 (size=53) 2024-11-16T04:24:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T04:24:26,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:24:26,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing add53441e0ab2ad8f2f1f109ed502c64, disabling compactions & flushes 2024-11-16T04:24:26,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. after waiting 0 ms 2024-11-16T04:24:26,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,343 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,343 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for add53441e0ab2ad8f2f1f109ed502c64: Waiting for close lock at 1731731066342Disabling compacts and flushes for region at 1731731066342Disabling writes for close at 1731731066342Writing region close event to WAL at 1731731066342Closed at 1731731066342 2024-11-16T04:24:26,345 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T04:24:26,345 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731731066345"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731731066345"}]},"ts":"1731731066345"} 2024-11-16T04:24:26,347 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T04:24:26,349 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T04:24:26,349 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731066349"}]},"ts":"1731731066349"} 2024-11-16T04:24:26,351 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T04:24:26,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, ASSIGN}] 2024-11-16T04:24:26,354 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, ASSIGN 2024-11-16T04:24:26,355 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, ASSIGN; state=OFFLINE, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=false 2024-11-16T04:24:26,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:26,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, ASSIGN because future has completed 2024-11-16T04:24:26,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:24:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T04:24:26,668 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,669 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:24:26,669 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,669 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:24:26,669 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,669 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,671 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,675 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add53441e0ab2ad8f2f1f109ed502c64 columnFamilyName A 2024-11-16T04:24:26,675 DEBUG [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:26,676 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(327): Store=add53441e0ab2ad8f2f1f109ed502c64/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:24:26,676 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,678 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add53441e0ab2ad8f2f1f109ed502c64 columnFamilyName B 2024-11-16T04:24:26,678 DEBUG [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:26,679 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(327): Store=add53441e0ab2ad8f2f1f109ed502c64/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:24:26,679 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,681 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add53441e0ab2ad8f2f1f109ed502c64 columnFamilyName C 2024-11-16T04:24:26,681 DEBUG [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:26,681 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(327): Store=add53441e0ab2ad8f2f1f109ed502c64/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:24:26,681 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,682 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,682 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,684 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,684 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,684 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:24:26,686 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,688 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:24:26,688 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened add53441e0ab2ad8f2f1f109ed502c64; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74668554, jitterRate=0.11264815926551819}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:24:26,688 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:26,689 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for add53441e0ab2ad8f2f1f109ed502c64: Running coprocessor pre-open hook at 1731731066670Writing region info on filesystem at 1731731066670Initializing all the Stores at 1731731066671 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731066671Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731066671Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731066671Cleaning up temporary data from old regions at 1731731066684 (+13 ms)Running coprocessor post-open hooks at 1731731066688 (+4 ms)Region opened successfully at 1731731066689 (+1 ms) 2024-11-16T04:24:26,690 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., pid=34, masterSystemTime=1731731066664 2024-11-16T04:24:26,692 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,692 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:26,692 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:26,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:24:26,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T04:24:26,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 in 185 msec 2024-11-16T04:24:26,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-16T04:24:26,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, ASSIGN in 344 msec 2024-11-16T04:24:26,699 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T04:24:26,700 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731066699"}]},"ts":"1731731066699"} 2024-11-16T04:24:26,701 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T04:24:26,702 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T04:24:26,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-16T04:24:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-16T04:24:27,663 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:27,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b33666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:27,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:27,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:27,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b44f32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:27,670 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:27,671 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@162cd357, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:27,671 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:27,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:27,679 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:24:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:24:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@358cd36f 2024-11-16T04:24:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:24:27,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:24:27,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:24:27,691 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:24:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:27,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741953_1129 (size=995) 2024-11-16T04:24:28,115 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T04:24:28,115 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T04:24:28,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:24:28,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, REOPEN/MOVE}] 2024-11-16T04:24:28,126 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, REOPEN/MOVE 2024-11-16T04:24:28,127 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, REOPEN/MOVE because future has completed 2024-11-16T04:24:28,129 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:24:28,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:24:28,282 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,282 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:24:28,282 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing add53441e0ab2ad8f2f1f109ed502c64, disabling compactions & flushes 2024-11-16T04:24:28,282 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,282 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,282 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. after waiting 0 ms 2024-11-16T04:24:28,282 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,286 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T04:24:28,287 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,287 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for add53441e0ab2ad8f2f1f109ed502c64: Waiting for close lock at 1731731068282Running coprocessor pre-close hooks at 1731731068282Disabling compacts and flushes for region at 1731731068282Disabling writes for close at 1731731068282Writing region close event to WAL at 1731731068283 (+1 ms)Running coprocessor post-close hooks at 1731731068287 (+4 ms)Closed at 1731731068287 2024-11-16T04:24:28,287 WARN [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: add53441e0ab2ad8f2f1f109ed502c64 to self. 2024-11-16T04:24:28,289 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=CLOSED 2024-11-16T04:24:28,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:24:28,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-16T04:24:28,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 in 163 msec 2024-11-16T04:24:28,294 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, REOPEN/MOVE; state=CLOSED, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=true 2024-11-16T04:24:28,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, REOPEN/MOVE because future has completed 2024-11-16T04:24:28,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:24:28,603 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,603 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:24:28,603 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,603 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:24:28,603 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,603 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,607 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,615 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add53441e0ab2ad8f2f1f109ed502c64 columnFamilyName A 2024-11-16T04:24:28,617 DEBUG [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:28,618 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(327): Store=add53441e0ab2ad8f2f1f109ed502c64/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:24:28,618 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,620 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add53441e0ab2ad8f2f1f109ed502c64 columnFamilyName B 2024-11-16T04:24:28,620 DEBUG [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:28,620 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(327): Store=add53441e0ab2ad8f2f1f109ed502c64/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:24:28,620 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,622 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region add53441e0ab2ad8f2f1f109ed502c64 columnFamilyName C 2024-11-16T04:24:28,622 DEBUG [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:28,622 INFO [StoreOpener-add53441e0ab2ad8f2f1f109ed502c64-1 {}] regionserver.HStore(327): Store=add53441e0ab2ad8f2f1f109ed502c64/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:24:28,622 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,623 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,625 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,626 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,626 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,627 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:24:28,628 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,629 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened add53441e0ab2ad8f2f1f109ed502c64; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70970822, jitterRate=0.05754765868186951}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:24:28,629 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,631 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for add53441e0ab2ad8f2f1f109ed502c64: Running coprocessor pre-open hook at 1731731068604Writing region info on filesystem at 1731731068604Initializing all the Stores at 1731731068604Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731068604Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731068604Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731068605 (+1 ms)Cleaning up temporary data from old regions at 1731731068626 (+21 ms)Running coprocessor post-open hooks at 1731731068629 (+3 ms)Region opened successfully at 1731731068630 (+1 ms) 2024-11-16T04:24:28,631 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., pid=39, masterSystemTime=1731731068599 2024-11-16T04:24:28,634 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,634 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,635 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=OPEN, openSeqNum=5, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:24:28,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-16T04:24:28,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 in 190 msec 2024-11-16T04:24:28,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-16T04:24:28,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, REOPEN/MOVE in 515 msec 2024-11-16T04:24:28,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T04:24:28,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-16T04:24:28,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 957 msec 2024-11-16T04:24:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-16T04:24:28,656 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:28,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3853aee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,658 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28055085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,660 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee4e7d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280b3bfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,663 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3afd1457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673614f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70646c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50287fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773a1f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@662f079f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32d4aac7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6255820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148de5e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,677 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,678 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34146cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,680 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35716ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba58e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,682 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14941d86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,684 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71597d91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721bdc4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,690 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2559f6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,693 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1542747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c4d062, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,696 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71cf67e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,698 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30d6385c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aba8a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:24:28,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:24:28,701 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:24:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:24:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:24:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@451c30ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,701 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:24:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:24:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:28,703 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:24:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2149b6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:24:28,705 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,705 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,706 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,708 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-16T04:24:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T04:24:28,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,711 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:28,712 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:28,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,717 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,717 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,733 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,733 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,734 DEBUG [Thread-655 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,734 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,734 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,734 DEBUG [Thread-655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,735 DEBUG [Thread-662 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,735 DEBUG [Thread-662 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,740 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:24:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:24:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:24:28,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:24:28,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:28,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:24:28,753 DEBUG [Thread-655 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:24:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T04:24:28,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ea26ac1f071f4e19b27dc37764029b7b_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731068743/Put/seqid=0 2024-11-16T04:24:28,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731128844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741954_1130 (size=17034) 2024-11-16T04:24:28,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:28,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:28,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:28,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:28,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731128848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731128848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731128854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731128849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ea26ac1f071f4e19b27dc37764029b7b_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea26ac1f071f4e19b27dc37764029b7b_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:28,883 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8593d11934f34b70851641f53616fcd9, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:28,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8593d11934f34b70851641f53616fcd9 is 175, key is test_row_0/A:col10/1731731068743/Put/seqid=0 2024-11-16T04:24:28,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741955_1131 (size=48139) 2024-11-16T04:24:28,941 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8593d11934f34b70851641f53616fcd9 2024-11-16T04:24:28,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731128963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/8b81b374fa1543a7b1314e57186cd93d is 50, key is test_row_0/B:col10/1731731068743/Put/seqid=0 2024-11-16T04:24:28,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731128973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731128974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741956_1132 (size=12001) 2024-11-16T04:24:28,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/8b81b374fa1543a7b1314e57186cd93d 2024-11-16T04:24:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731128984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:28,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731128984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T04:24:29,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5 is 50, key is test_row_0/C:col10/1731731068743/Put/seqid=0 2024-11-16T04:24:29,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741957_1133 (size=12001) 2024-11-16T04:24:29,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5 2024-11-16T04:24:29,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8593d11934f34b70851641f53616fcd9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9 2024-11-16T04:24:29,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9, entries=250, sequenceid=15, filesize=47.0 K 2024-11-16T04:24:29,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/8b81b374fa1543a7b1314e57186cd93d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/8b81b374fa1543a7b1314e57186cd93d 2024-11-16T04:24:29,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/8b81b374fa1543a7b1314e57186cd93d, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T04:24:29,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5 2024-11-16T04:24:29,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T04:24:29,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for add53441e0ab2ad8f2f1f109ed502c64 in 352ms, sequenceid=15, compaction requested=false 2024-11-16T04:24:29,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:29,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T04:24:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731129195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731129197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731129199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731129204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731129205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e67c55177adc412a8e893bb050c9c615_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731069179/Put/seqid=0 2024-11-16T04:24:29,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741958_1134 (size=14594) 2024-11-16T04:24:29,259 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:29,267 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e67c55177adc412a8e893bb050c9c615_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e67c55177adc412a8e893bb050c9c615_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:29,274 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/865b929b28354b918df2fa96c3a1d5a0, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:29,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/865b929b28354b918df2fa96c3a1d5a0 is 175, key is test_row_0/A:col10/1731731069179/Put/seqid=0 2024-11-16T04:24:29,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741959_1135 (size=39549) 2024-11-16T04:24:29,315 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/865b929b28354b918df2fa96c3a1d5a0 2024-11-16T04:24:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731129314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731129314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731129315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731129316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731129316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T04:24:29,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/a2e88bd4d35a4d489571ca045a6d9fba is 50, key is test_row_0/B:col10/1731731069179/Put/seqid=0 2024-11-16T04:24:29,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741960_1136 (size=12001) 2024-11-16T04:24:29,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731129524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731129524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731129533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731129534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731129534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/a2e88bd4d35a4d489571ca045a6d9fba 2024-11-16T04:24:29,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/24382fcb28e644869902364272973d55 is 50, key is test_row_0/C:col10/1731731069179/Put/seqid=0 2024-11-16T04:24:29,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741961_1137 (size=12001) 2024-11-16T04:24:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T04:24:29,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731129843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731129844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731129854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731129854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731129854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:29,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:29,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:30,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:30,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:30,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:30,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/24382fcb28e644869902364272973d55 2024-11-16T04:24:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/865b929b28354b918df2fa96c3a1d5a0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0 2024-11-16T04:24:30,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0, entries=200, sequenceid=41, filesize=38.6 K 2024-11-16T04:24:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/a2e88bd4d35a4d489571ca045a6d9fba as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a2e88bd4d35a4d489571ca045a6d9fba 2024-11-16T04:24:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a2e88bd4d35a4d489571ca045a6d9fba, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T04:24:30,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/24382fcb28e644869902364272973d55 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/24382fcb28e644869902364272973d55 2024-11-16T04:24:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/24382fcb28e644869902364272973d55, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T04:24:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for add53441e0ab2ad8f2f1f109ed502c64 in 1066ms, sequenceid=41, compaction requested=false 2024-11-16T04:24:30,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-16T04:24:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:30,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T04:24:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168804e64462354c2d8cd61413a7fef879_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_1/A:col10/1731731069201/Put/seqid=0 2024-11-16T04:24:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741962_1138 (size=9714) 2024-11-16T04:24:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168804e64462354c2d8cd61413a7fef879_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168804e64462354c2d8cd61413a7fef879_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/acc9f002ffb2460c8e89a1fc54665fe0, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/acc9f002ffb2460c8e89a1fc54665fe0 is 175, key is test_row_1/A:col10/1731731069201/Put/seqid=0 2024-11-16T04:24:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741963_1139 (size=22361) 2024-11-16T04:24:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/acc9f002ffb2460c8e89a1fc54665fe0 2024-11-16T04:24:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/191ab5d50cdb4b51ae7d7fb7cd6b3146 is 50, key is test_row_1/B:col10/1731731069201/Put/seqid=0 2024-11-16T04:24:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741964_1140 (size=9657) 2024-11-16T04:24:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,340 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/191ab5d50cdb4b51ae7d7fb7cd6b3146 2024-11-16T04:24:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/395337ce8feb40e79a6f9ccdcbd03de7 is 50, key is test_row_1/C:col10/1731731069201/Put/seqid=0 2024-11-16T04:24:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:30,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741965_1141 (size=9657) 2024-11-16T04:24:30,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/395337ce8feb40e79a6f9ccdcbd03de7 2024-11-16T04:24:30,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/acc9f002ffb2460c8e89a1fc54665fe0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0 2024-11-16T04:24:30,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,400 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0, entries=100, sequenceid=51, filesize=21.8 K 2024-11-16T04:24:30,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/191ab5d50cdb4b51ae7d7fb7cd6b3146 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/191ab5d50cdb4b51ae7d7fb7cd6b3146 2024-11-16T04:24:30,404 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:24:30,411 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/191ab5d50cdb4b51ae7d7fb7cd6b3146, entries=100, sequenceid=51, filesize=9.4 K 2024-11-16T04:24:30,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/395337ce8feb40e79a6f9ccdcbd03de7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/395337ce8feb40e79a6f9ccdcbd03de7 2024-11-16T04:24:30,419 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/395337ce8feb40e79a6f9ccdcbd03de7, entries=100, sequenceid=51, filesize=9.4 K 2024-11-16T04:24:30,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=80.51 KB/82440 for add53441e0ab2ad8f2f1f109ed502c64 in 161ms, sequenceid=51, compaction requested=true 2024-11-16T04:24:30,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:30,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:30,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-16T04:24:30,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:30,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T04:24:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-16T04:24:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-16T04:24:30,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-16T04:24:30,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7310 sec 2024-11-16T04:24:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116af5919bb71a24887baae168a1dea288a_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731130494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731130496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731130497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731130499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731130507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741967_1143 (size=34118) 2024-11-16T04:24:30,565 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:30,577 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116af5919bb71a24887baae168a1dea288a_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116af5919bb71a24887baae168a1dea288a_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:30,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/9c81c9b9a8434b268d47b7e79ad29334, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:30,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/9c81c9b9a8434b268d47b7e79ad29334 is 175, key is test_row_0/A:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:30,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741966_1142 (size=108362) 2024-11-16T04:24:30,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=67, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/9c81c9b9a8434b268d47b7e79ad29334 2024-11-16T04:24:30,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/b9072f1d99774f1182b8f9ee8a7ec75c is 50, key is test_row_0/B:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:30,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731130614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731130614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731130623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731130623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731130634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741968_1144 (size=12001) 2024-11-16T04:24:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731130824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731130834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731130834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731130834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:30,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731130844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-16T04:24:30,853 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:30,854 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-16T04:24:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T04:24:30,856 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:30,858 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:30,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T04:24:31,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T04:24:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/b9072f1d99774f1182b8f9ee8a7ec75c 2024-11-16T04:24:31,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/f762ad79134044fa8c205fee363feb8c is 50, key is test_row_0/C:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:31,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741969_1145 (size=12001) 2024-11-16T04:24:31,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731131134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731131143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731131143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731131144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731131154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T04:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T04:24:31,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T04:24:31,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T04:24:31,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:31,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T04:24:31,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/f762ad79134044fa8c205fee363feb8c 2024-11-16T04:24:31,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/9c81c9b9a8434b268d47b7e79ad29334 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334 2024-11-16T04:24:31,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334, entries=600, sequenceid=67, filesize=105.8 K 2024-11-16T04:24:31,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/b9072f1d99774f1182b8f9ee8a7ec75c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b9072f1d99774f1182b8f9ee8a7ec75c 2024-11-16T04:24:31,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b9072f1d99774f1182b8f9ee8a7ec75c, entries=150, sequenceid=67, filesize=11.7 K 2024-11-16T04:24:31,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/f762ad79134044fa8c205fee363feb8c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/f762ad79134044fa8c205fee363feb8c 2024-11-16T04:24:31,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/f762ad79134044fa8c205fee363feb8c, entries=150, sequenceid=67, filesize=11.7 K 2024-11-16T04:24:31,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for add53441e0ab2ad8f2f1f109ed502c64 in 1132ms, sequenceid=67, compaction requested=true 2024-11-16T04:24:31,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:31,554 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:31,554 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:31,558 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 218411 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:31,558 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:31,558 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:31,558 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:31,558 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,558 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,558 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/8b81b374fa1543a7b1314e57186cd93d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a2e88bd4d35a4d489571ca045a6d9fba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/191ab5d50cdb4b51ae7d7fb7cd6b3146, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b9072f1d99774f1182b8f9ee8a7ec75c] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=44.6 K 2024-11-16T04:24:31,558 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=213.3 K 2024-11-16T04:24:31,558 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,559 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334] 2024-11-16T04:24:31,559 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b81b374fa1543a7b1314e57186cd93d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731731068722 2024-11-16T04:24:31,560 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e88bd4d35a4d489571ca045a6d9fba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731731068843 2024-11-16T04:24:31,560 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8593d11934f34b70851641f53616fcd9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731731068722 2024-11-16T04:24:31,560 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 191ab5d50cdb4b51ae7d7fb7cd6b3146, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731069199 2024-11-16T04:24:31,560 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 865b929b28354b918df2fa96c3a1d5a0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731731068843 2024-11-16T04:24:31,561 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting acc9f002ffb2460c8e89a1fc54665fe0, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731069199 2024-11-16T04:24:31,561 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b9072f1d99774f1182b8f9ee8a7ec75c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731731070401 2024-11-16T04:24:31,561 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c81c9b9a8434b268d47b7e79ad29334, keycount=600, bloomtype=ROW, size=105.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731731070375 2024-11-16T04:24:31,581 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:31,582 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/71524e39404141978b78dfdaa2cf5d2c is 50, key is test_row_0/B:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:31,589 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:31,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741970_1146 (size=12139) 2024-11-16T04:24:31,621 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f23842dcfd3a48faa6b8a8422652b648_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:31,626 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f23842dcfd3a48faa6b8a8422652b648_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:31,626 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f23842dcfd3a48faa6b8a8422652b648_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:31,630 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/71524e39404141978b78dfdaa2cf5d2c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/71524e39404141978b78dfdaa2cf5d2c 2024-11-16T04:24:31,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-16T04:24:31,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T04:24:31,643 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into 71524e39404141978b78dfdaa2cf5d2c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:31,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:31,643 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=12, startTime=1731731071554; duration=0sec 2024-11-16T04:24:31,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:31,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:31,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:31,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:31,648 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:31,648 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:31,648 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:31,649 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/24382fcb28e644869902364272973d55, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/395337ce8feb40e79a6f9ccdcbd03de7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/f762ad79134044fa8c205fee363feb8c] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=44.6 K 2024-11-16T04:24:31,649 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e0e55d2eca4dd9b15c2a5bd3ddc1f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731731068722 2024-11-16T04:24:31,650 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 24382fcb28e644869902364272973d55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731731068843 2024-11-16T04:24:31,652 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 395337ce8feb40e79a6f9ccdcbd03de7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731069199 2024-11-16T04:24:31,653 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f762ad79134044fa8c205fee363feb8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731731070401 2024-11-16T04:24:31,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164d62fa93a61643c2b63e8d051f08a147_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731070477/Put/seqid=0 2024-11-16T04:24:31,673 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:31,673 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/49a52e1447904fd8bd04816c43077f76 is 50, key is test_row_0/C:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:31,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741971_1147 (size=4469) 2024-11-16T04:24:31,682 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#127 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:31,682 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4b934f6f85f94ae7a4bbfa679a120efe is 175, key is test_row_0/A:col10/1731731070401/Put/seqid=0 2024-11-16T04:24:31,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741972_1148 (size=12154) 2024-11-16T04:24:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731131684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731131686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731131690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731131693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731131698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741973_1149 (size=12139) 2024-11-16T04:24:31,732 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/49a52e1447904fd8bd04816c43077f76 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/49a52e1447904fd8bd04816c43077f76 2024-11-16T04:24:31,742 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into 49a52e1447904fd8bd04816c43077f76(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:31,742 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:31,742 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=12, startTime=1731731071554; duration=0sec 2024-11-16T04:24:31,742 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:31,742 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:31,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741974_1150 (size=31093) 2024-11-16T04:24:31,756 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4b934f6f85f94ae7a4bbfa679a120efe as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4b934f6f85f94ae7a4bbfa679a120efe 2024-11-16T04:24:31,766 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into 4b934f6f85f94ae7a4bbfa679a120efe(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:31,766 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:31,766 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=12, startTime=1731731071553; duration=0sec 2024-11-16T04:24:31,766 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:31,766 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731131804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731131804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731131814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731131814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:31,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731131815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:31,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T04:24:32,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731132013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731132024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731132024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731132024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731132024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:32,090 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164d62fa93a61643c2b63e8d051f08a147_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164d62fa93a61643c2b63e8d051f08a147_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:32,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/449f49417b1d4118aa44be9816321ea4, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:32,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/449f49417b1d4118aa44be9816321ea4 is 175, key is test_row_0/A:col10/1731731070477/Put/seqid=0 2024-11-16T04:24:32,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741975_1151 (size=30955) 2024-11-16T04:24:32,133 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/449f49417b1d4118aa44be9816321ea4 2024-11-16T04:24:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/a8e141f8c67a4598a69843e2f769a00a is 50, key is test_row_0/B:col10/1731731070477/Put/seqid=0 2024-11-16T04:24:32,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741976_1152 (size=12001) 2024-11-16T04:24:32,167 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/a8e141f8c67a4598a69843e2f769a00a 2024-11-16T04:24:32,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/18f7acd82f2040e7ad385ff01a8338f1 is 50, key is test_row_0/C:col10/1731731070477/Put/seqid=0 2024-11-16T04:24:32,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741977_1153 (size=12001) 2024-11-16T04:24:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731132324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731132333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731132333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731132333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731132334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,601 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/18f7acd82f2040e7ad385ff01a8338f1 2024-11-16T04:24:32,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/449f49417b1d4118aa44be9816321ea4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4 2024-11-16T04:24:32,622 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4, entries=150, sequenceid=87, filesize=30.2 K 2024-11-16T04:24:32,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/a8e141f8c67a4598a69843e2f769a00a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a8e141f8c67a4598a69843e2f769a00a 2024-11-16T04:24:32,628 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a8e141f8c67a4598a69843e2f769a00a, entries=150, sequenceid=87, filesize=11.7 K 2024-11-16T04:24:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/18f7acd82f2040e7ad385ff01a8338f1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/18f7acd82f2040e7ad385ff01a8338f1 2024-11-16T04:24:32,638 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/18f7acd82f2040e7ad385ff01a8338f1, entries=150, sequenceid=87, filesize=11.7 K 2024-11-16T04:24:32,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for add53441e0ab2ad8f2f1f109ed502c64 in 1006ms, sequenceid=87, compaction requested=false 2024-11-16T04:24:32,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:32,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:32,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-16T04:24:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-16T04:24:32,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T04:24:32,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7840 sec 2024-11-16T04:24:32,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7920 sec 2024-11-16T04:24:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:32,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T04:24:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:32,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111623d8be92692f4e4a87f42188a1fb1f19_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741978_1154 (size=12154) 2024-11-16T04:24:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731132880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731132881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731132884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731132886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:32,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731132894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-16T04:24:33,004 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:33,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-16T04:24:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T04:24:33,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:33,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:33,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:33,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731133004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731133004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731133004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731133004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731133013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T04:24:33,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T04:24:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731133213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731133223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731133223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731133223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731133224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:33,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111623d8be92692f4e4a87f42188a1fb1f19_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111623d8be92692f4e4a87f42188a1fb1f19_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:33,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/24626f0ca39d4fd4b16bd0974f23d98f, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:33,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/24626f0ca39d4fd4b16bd0974f23d98f is 175, key is test_row_0/A:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:33,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741979_1155 (size=30955) 2024-11-16T04:24:33,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/24626f0ca39d4fd4b16bd0974f23d98f 2024-11-16T04:24:33,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T04:24:33,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:33,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/06ec234dcbf843e987ef8d35c94b1b0d is 50, key is test_row_0/B:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T04:24:33,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741980_1156 (size=12001) 2024-11-16T04:24:33,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/06ec234dcbf843e987ef8d35c94b1b0d 2024-11-16T04:24:33,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/b8bf5737f7c04e649a8aee6a0b5d3bd7 is 50, key is test_row_0/C:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:33,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741981_1157 (size=12001) 2024-11-16T04:24:33,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T04:24:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731133523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731133533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731133533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731133534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731133534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:33,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T04:24:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T04:24:33,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/b8bf5737f7c04e649a8aee6a0b5d3bd7 2024-11-16T04:24:33,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T04:24:33,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:33,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/24626f0ca39d4fd4b16bd0974f23d98f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f 2024-11-16T04:24:33,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f, entries=150, sequenceid=107, filesize=30.2 K 2024-11-16T04:24:33,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/06ec234dcbf843e987ef8d35c94b1b0d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/06ec234dcbf843e987ef8d35c94b1b0d 2024-11-16T04:24:33,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/06ec234dcbf843e987ef8d35c94b1b0d, entries=150, sequenceid=107, filesize=11.7 K 2024-11-16T04:24:33,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/b8bf5737f7c04e649a8aee6a0b5d3bd7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b8bf5737f7c04e649a8aee6a0b5d3bd7 2024-11-16T04:24:33,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b8bf5737f7c04e649a8aee6a0b5d3bd7, entries=150, sequenceid=107, filesize=11.7 K 2024-11-16T04:24:33,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for add53441e0ab2ad8f2f1f109ed502c64 in 993ms, sequenceid=107, compaction requested=true 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:33,832 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:33,832 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:33,833 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:33,833 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:33,833 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:33,833 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:33,833 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,833 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,833 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/71524e39404141978b78dfdaa2cf5d2c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a8e141f8c67a4598a69843e2f769a00a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/06ec234dcbf843e987ef8d35c94b1b0d] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=35.3 K 2024-11-16T04:24:33,833 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4b934f6f85f94ae7a4bbfa679a120efe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=90.8 K 2024-11-16T04:24:33,833 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,833 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4b934f6f85f94ae7a4bbfa679a120efe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f] 2024-11-16T04:24:33,834 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 71524e39404141978b78dfdaa2cf5d2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731731070401 2024-11-16T04:24:33,834 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b934f6f85f94ae7a4bbfa679a120efe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731731070401 2024-11-16T04:24:33,834 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e141f8c67a4598a69843e2f769a00a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731731070472 2024-11-16T04:24:33,834 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449f49417b1d4118aa44be9816321ea4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731731070472 2024-11-16T04:24:33,834 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ec234dcbf843e987ef8d35c94b1b0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731731071688 2024-11-16T04:24:33,834 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24626f0ca39d4fd4b16bd0974f23d98f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731731071688 2024-11-16T04:24:33,853 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:33,855 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:33,855 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/b18b25569b2342dca739bbab052b565c is 50, key is test_row_0/B:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:33,856 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e71f2e932ceb46c1827f4f09e174bf31_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:33,859 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e71f2e932ceb46c1827f4f09e174bf31_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:33,859 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e71f2e932ceb46c1827f4f09e174bf31_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:33,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741982_1158 (size=12241) 2024-11-16T04:24:33,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741983_1159 (size=4469) 2024-11-16T04:24:33,865 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#136 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:33,865 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4858198f9a58408c8d1a8eb07300726c is 175, key is test_row_0/A:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:33,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741984_1160 (size=31195) 2024-11-16T04:24:33,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-16T04:24:33,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:33,955 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T04:24:33,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f90e3e3fdf574858b7e647de892a49f5_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731072877/Put/seqid=0 2024-11-16T04:24:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741985_1161 (size=12154) 2024-11-16T04:24:33,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:33,975 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f90e3e3fdf574858b7e647de892a49f5_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f90e3e3fdf574858b7e647de892a49f5_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/407e9f6091be44d8b3168035593c6fd3, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:33,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/407e9f6091be44d8b3168035593c6fd3 is 175, key is test_row_0/A:col10/1731731072877/Put/seqid=0 2024-11-16T04:24:34,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741986_1162 (size=30955) 2024-11-16T04:24:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:34,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:34,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731134056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731134057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731134058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731134059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731134060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T04:24:34,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731134163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731134174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731134174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731134174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731134174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,269 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/b18b25569b2342dca739bbab052b565c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b18b25569b2342dca739bbab052b565c 2024-11-16T04:24:34,275 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into b18b25569b2342dca739bbab052b565c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:34,275 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:34,275 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=13, startTime=1731731073832; duration=0sec 2024-11-16T04:24:34,276 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:34,276 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:34,276 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:34,277 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:34,277 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:34,277 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:34,277 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/49a52e1447904fd8bd04816c43077f76, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/18f7acd82f2040e7ad385ff01a8338f1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b8bf5737f7c04e649a8aee6a0b5d3bd7] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=35.3 K 2024-11-16T04:24:34,278 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a52e1447904fd8bd04816c43077f76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731731070401 2024-11-16T04:24:34,278 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f7acd82f2040e7ad385ff01a8338f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731731070472 2024-11-16T04:24:34,279 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b8bf5737f7c04e649a8aee6a0b5d3bd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731731071688 2024-11-16T04:24:34,297 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:34,298 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/53e3eb841b6b4c3a8bca18f0800a94a8 is 50, key is test_row_0/C:col10/1731731071690/Put/seqid=0 2024-11-16T04:24:34,303 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4858198f9a58408c8d1a8eb07300726c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4858198f9a58408c8d1a8eb07300726c 2024-11-16T04:24:34,312 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into 4858198f9a58408c8d1a8eb07300726c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:34,313 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:34,313 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=13, startTime=1731731073832; duration=0sec 2024-11-16T04:24:34,313 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:34,313 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:34,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741987_1163 (size=12241) 2024-11-16T04:24:34,355 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/53e3eb841b6b4c3a8bca18f0800a94a8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/53e3eb841b6b4c3a8bca18f0800a94a8 2024-11-16T04:24:34,362 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into 53e3eb841b6b4c3a8bca18f0800a94a8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:34,362 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:34,362 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=13, startTime=1731731073832; duration=0sec 2024-11-16T04:24:34,362 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:34,362 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:34,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731134384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731134393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731134393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731134394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731134396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,411 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/407e9f6091be44d8b3168035593c6fd3 2024-11-16T04:24:34,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/f577ac913cff46069e58ab5a311b4dce is 50, key is test_row_0/B:col10/1731731072877/Put/seqid=0 2024-11-16T04:24:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741988_1164 (size=12001) 2024-11-16T04:24:34,467 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/f577ac913cff46069e58ab5a311b4dce 2024-11-16T04:24:34,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/7411ec2c3fdb4036ba8c599f719e0a8b is 50, key is test_row_0/C:col10/1731731072877/Put/seqid=0 2024-11-16T04:24:34,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741989_1165 (size=12001) 2024-11-16T04:24:34,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731134694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731134703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731134703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731134703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731134709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:34,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:34,906 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/7411ec2c3fdb4036ba8c599f719e0a8b 2024-11-16T04:24:34,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/407e9f6091be44d8b3168035593c6fd3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3 2024-11-16T04:24:34,915 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3, entries=150, sequenceid=127, filesize=30.2 K 2024-11-16T04:24:34,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/f577ac913cff46069e58ab5a311b4dce as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/f577ac913cff46069e58ab5a311b4dce 2024-11-16T04:24:34,920 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/f577ac913cff46069e58ab5a311b4dce, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T04:24:34,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/7411ec2c3fdb4036ba8c599f719e0a8b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7411ec2c3fdb4036ba8c599f719e0a8b 2024-11-16T04:24:34,926 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7411ec2c3fdb4036ba8c599f719e0a8b, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T04:24:34,927 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for add53441e0ab2ad8f2f1f109ed502c64 in 972ms, sequenceid=127, compaction requested=false 2024-11-16T04:24:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-16T04:24:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-16T04:24:34,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-16T04:24:34,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9210 sec 2024-11-16T04:24:34,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.9270 sec 2024-11-16T04:24:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-16T04:24:35,153 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:35,154 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T04:24:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:35,155 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:35,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:35,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:35,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:35,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-16T04:24:35,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d21e622c2fa34b8fba57dba99702538e_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741990_1166 (size=14794) 2024-11-16T04:24:35,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:35,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d21e622c2fa34b8fba57dba99702538e_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d21e622c2fa34b8fba57dba99702538e_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:35,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/2e78ecbdecf54f54a12fbea0cc95cb9a, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:35,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/2e78ecbdecf54f54a12fbea0cc95cb9a is 175, key is test_row_0/A:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731135230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731135236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731135237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731135238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731135239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741991_1167 (size=39749) 2024-11-16T04:24:35,245 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/2e78ecbdecf54f54a12fbea0cc95cb9a 2024-11-16T04:24:35,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/49b389bdc3a94f02a99d67dc371e5776 is 50, key is test_row_0/B:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:35,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741992_1168 (size=12151) 2024-11-16T04:24:35,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/49b389bdc3a94f02a99d67dc371e5776 2024-11-16T04:24:35,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/d16ec634d9e44263a5994cc070f57a82 is 50, key is test_row_0/C:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741993_1169 (size=12151) 2024-11-16T04:24:35,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/d16ec634d9e44263a5994cc070f57a82 2024-11-16T04:24:35,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/2e78ecbdecf54f54a12fbea0cc95cb9a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a 2024-11-16T04:24:35,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a, entries=200, sequenceid=147, filesize=38.8 K 2024-11-16T04:24:35,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/49b389bdc3a94f02a99d67dc371e5776 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/49b389bdc3a94f02a99d67dc371e5776 2024-11-16T04:24:35,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/49b389bdc3a94f02a99d67dc371e5776, entries=150, sequenceid=147, filesize=11.9 K 2024-11-16T04:24:35,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/d16ec634d9e44263a5994cc070f57a82 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d16ec634d9e44263a5994cc070f57a82 2024-11-16T04:24:35,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d16ec634d9e44263a5994cc070f57a82, entries=150, sequenceid=147, filesize=11.9 K 2024-11-16T04:24:35,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for add53441e0ab2ad8f2f1f109ed502c64 in 106ms, sequenceid=147, compaction requested=true 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:35,314 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:35,314 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:35,316 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:35,316 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:35,316 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:35,316 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:35,317 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,317 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4858198f9a58408c8d1a8eb07300726c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=99.5 K 2024-11-16T04:24:35,317 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,317 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4858198f9a58408c8d1a8eb07300726c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a] 2024-11-16T04:24:35,317 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,317 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b18b25569b2342dca739bbab052b565c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/f577ac913cff46069e58ab5a311b4dce, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/49b389bdc3a94f02a99d67dc371e5776] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=35.5 K 2024-11-16T04:24:35,318 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b18b25569b2342dca739bbab052b565c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731731071688 2024-11-16T04:24:35,318 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4858198f9a58408c8d1a8eb07300726c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731731071688 2024-11-16T04:24:35,319 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 407e9f6091be44d8b3168035593c6fd3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731731072877 2024-11-16T04:24:35,319 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f577ac913cff46069e58ab5a311b4dce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731731072877 2024-11-16T04:24:35,319 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e78ecbdecf54f54a12fbea0cc95cb9a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731731074055 2024-11-16T04:24:35,319 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 49b389bdc3a94f02a99d67dc371e5776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731731074055 2024-11-16T04:24:35,329 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:35,341 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116968ca91ceb3c4128afdbb022581755a3_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:35,342 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:35,344 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116968ca91ceb3c4128afdbb022581755a3_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:35,344 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116968ca91ceb3c4128afdbb022581755a3_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:35,345 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/c5196a1480fa42f896556ad0cf27e773 is 50, key is test_row_0/B:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:35,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:35,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T04:24:35,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741994_1170 (size=4469) 2024-11-16T04:24:35,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116579ad89432fe46a79eb2cf1c28a88e26_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731075345/Put/seqid=0 2024-11-16T04:24:35,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741995_1171 (size=12493) 2024-11-16T04:24:35,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741996_1172 (size=14794) 2024-11-16T04:24:35,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731135397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731135399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731135401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731135402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731135403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:35,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731135513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731135514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731135514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731135514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731135514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:35,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:35,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731135724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731135724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731135724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731135724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731135724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:35,764 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#144 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:35,765 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/5a34102eb2eb4472999d939e915b87f6 is 175, key is test_row_0/A:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741997_1173 (size=31447) 2024-11-16T04:24:35,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:35,780 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/c5196a1480fa42f896556ad0cf27e773 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c5196a1480fa42f896556ad0cf27e773 2024-11-16T04:24:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:35,786 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116579ad89432fe46a79eb2cf1c28a88e26_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116579ad89432fe46a79eb2cf1c28a88e26_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:35,788 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/5a34102eb2eb4472999d939e915b87f6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/5a34102eb2eb4472999d939e915b87f6 2024-11-16T04:24:35,789 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c105fd14ebdd4949914d19a1c504bf0d, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:35,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c105fd14ebdd4949914d19a1c504bf0d is 175, key is test_row_0/A:col10/1731731075345/Put/seqid=0 2024-11-16T04:24:35,790 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into c5196a1480fa42f896556ad0cf27e773(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:35,790 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:35,791 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=13, startTime=1731731075314; duration=0sec 2024-11-16T04:24:35,791 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:35,791 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:35,791 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:35,793 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:35,793 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:35,793 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,794 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/53e3eb841b6b4c3a8bca18f0800a94a8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7411ec2c3fdb4036ba8c599f719e0a8b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d16ec634d9e44263a5994cc070f57a82] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=35.5 K 2024-11-16T04:24:35,795 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e3eb841b6b4c3a8bca18f0800a94a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731731071688 2024-11-16T04:24:35,796 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7411ec2c3fdb4036ba8c599f719e0a8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731731072877 2024-11-16T04:24:35,796 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d16ec634d9e44263a5994cc070f57a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731731074055 2024-11-16T04:24:35,799 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into 5a34102eb2eb4472999d939e915b87f6(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:35,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:35,799 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=13, startTime=1731731075314; duration=0sec 2024-11-16T04:24:35,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:35,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:35,822 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:35,823 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/b223720d136747579c3a03edd77e6502 is 50, key is test_row_0/C:col10/1731731075205/Put/seqid=0 2024-11-16T04:24:35,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741998_1174 (size=39749) 2024-11-16T04:24:35,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741999_1175 (size=12493) 2024-11-16T04:24:35,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:35,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731136033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731136034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731136034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731136035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731136035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c105fd14ebdd4949914d19a1c504bf0d 2024-11-16T04:24:36,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/86c59b88f3ca42689550878295c2b257 is 50, key is test_row_0/B:col10/1731731075345/Put/seqid=0 2024-11-16T04:24:36,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,253 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/b223720d136747579c3a03edd77e6502 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b223720d136747579c3a03edd77e6502 2024-11-16T04:24:36,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742000_1176 (size=12151) 2024-11-16T04:24:36,261 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into b223720d136747579c3a03edd77e6502(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:36,261 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:36,261 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=13, startTime=1731731075314; duration=0sec 2024-11-16T04:24:36,261 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:36,261 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:36,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731136543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:36,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731136544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731136544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731136544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731136544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:36,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:36,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/86c59b88f3ca42689550878295c2b257 2024-11-16T04:24:36,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/968dafcec3e74c588e3730fb315d6d00 is 50, key is test_row_0/C:col10/1731731075345/Put/seqid=0 2024-11-16T04:24:36,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742001_1177 (size=12151) 2024-11-16T04:24:36,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:36,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:37,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:37,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:37,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/968dafcec3e74c588e3730fb315d6d00 2024-11-16T04:24:37,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c105fd14ebdd4949914d19a1c504bf0d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d 2024-11-16T04:24:37,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d, entries=200, sequenceid=168, filesize=38.8 K 2024-11-16T04:24:37,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/86c59b88f3ca42689550878295c2b257 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/86c59b88f3ca42689550878295c2b257 2024-11-16T04:24:37,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/86c59b88f3ca42689550878295c2b257, entries=150, sequenceid=168, filesize=11.9 K 2024-11-16T04:24:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/968dafcec3e74c588e3730fb315d6d00 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/968dafcec3e74c588e3730fb315d6d00 2024-11-16T04:24:37,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/968dafcec3e74c588e3730fb315d6d00, entries=150, sequenceid=168, filesize=11.9 K 2024-11-16T04:24:37,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for add53441e0ab2ad8f2f1f109ed502c64 in 1766ms, sequenceid=168, compaction requested=false 2024-11-16T04:24:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:37,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T04:24:37,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,159 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:24:37,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168a01d250da2c454bb5ea2f341fedeeb8_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731075402/Put/seqid=0 2024-11-16T04:24:37,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742002_1178 (size=12304) 2024-11-16T04:24:37,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,207 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168a01d250da2c454bb5ea2f341fedeeb8_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a01d250da2c454bb5ea2f341fedeeb8_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:37,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/fbc90b6e839e4861bc3e3aff87278f42, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:37,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/fbc90b6e839e4861bc3e3aff87278f42 is 175, key is test_row_0/A:col10/1731731075402/Put/seqid=0 2024-11-16T04:24:37,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742003_1179 (size=31105) 2024-11-16T04:24:37,223 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/fbc90b6e839e4861bc3e3aff87278f42 2024-11-16T04:24:37,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/c8dae0aa35094e88876603d2e8977f33 is 50, key is test_row_0/B:col10/1731731075402/Put/seqid=0 2024-11-16T04:24:37,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742004_1180 (size=12151) 2024-11-16T04:24:37,247 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/c8dae0aa35094e88876603d2e8977f33 2024-11-16T04:24:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/a588a3eacbb449889b400410800b2c43 is 50, key is test_row_0/C:col10/1731731075402/Put/seqid=0 2024-11-16T04:24:37,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742005_1181 (size=12151) 2024-11-16T04:24:37,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/a588a3eacbb449889b400410800b2c43 2024-11-16T04:24:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/fbc90b6e839e4861bc3e3aff87278f42 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42 2024-11-16T04:24:37,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42, entries=150, sequenceid=186, filesize=30.4 K 2024-11-16T04:24:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/c8dae0aa35094e88876603d2e8977f33 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c8dae0aa35094e88876603d2e8977f33 2024-11-16T04:24:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,292 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c8dae0aa35094e88876603d2e8977f33, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T04:24:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/a588a3eacbb449889b400410800b2c43 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/a588a3eacbb449889b400410800b2c43 2024-11-16T04:24:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,306 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/a588a3eacbb449889b400410800b2c43, entries=150, sequenceid=186, filesize=11.9 K 2024-11-16T04:24:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for add53441e0ab2ad8f2f1f109ed502c64 in 149ms, sequenceid=186, compaction requested=true 2024-11-16T04:24:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T04:24:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T04:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T04:24:37,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1540 sec 2024-11-16T04:24:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1590 sec 2024-11-16T04:24:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:37,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:37,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:24:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116efd3fdf0f3dc46aeb5d574ffb5a20b4b_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731137659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731137659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731137661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731137662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731137672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742007_1183 (size=27248) 2024-11-16T04:24:37,702 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:37,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116efd3fdf0f3dc46aeb5d574ffb5a20b4b_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efd3fdf0f3dc46aeb5d574ffb5a20b4b_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:37,713 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/0bc6bc40000542eebfa0eb9b7a4cdb4e, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:37,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/0bc6bc40000542eebfa0eb9b7a4cdb4e is 175, key is test_row_0/A:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742006_1182 (size=83035) 2024-11-16T04:24:37,717 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/0bc6bc40000542eebfa0eb9b7a4cdb4e 2024-11-16T04:24:37,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/110b1a4114024ae49fdaa197b2dbe06f is 50, key is test_row_0/B:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742008_1184 (size=12151) 2024-11-16T04:24:37,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/110b1a4114024ae49fdaa197b2dbe06f 2024-11-16T04:24:37,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/fd04c994e62d4e51b453b8bc2b2ae187 is 50, key is test_row_0/C:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742009_1185 (size=12151) 2024-11-16T04:24:37,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/fd04c994e62d4e51b453b8bc2b2ae187 2024-11-16T04:24:37,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731137774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731137774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731137774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/0bc6bc40000542eebfa0eb9b7a4cdb4e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e 2024-11-16T04:24:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731137783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731137783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e, entries=450, sequenceid=198, filesize=81.1 K 2024-11-16T04:24:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:37,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:37,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/110b1a4114024ae49fdaa197b2dbe06f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/110b1a4114024ae49fdaa197b2dbe06f 2024-11-16T04:24:37,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/110b1a4114024ae49fdaa197b2dbe06f, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:24:37,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/fd04c994e62d4e51b453b8bc2b2ae187 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/fd04c994e62d4e51b453b8bc2b2ae187 2024-11-16T04:24:37,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/fd04c994e62d4e51b453b8bc2b2ae187, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:24:37,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for add53441e0ab2ad8f2f1f109ed502c64 in 197ms, sequenceid=198, compaction requested=true 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:37,808 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:37,808 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:37,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:37,811 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 185336 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:37,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:37,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:37,811 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:37,811 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,811 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c5196a1480fa42f896556ad0cf27e773, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/86c59b88f3ca42689550878295c2b257, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c8dae0aa35094e88876603d2e8977f33, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/110b1a4114024ae49fdaa197b2dbe06f] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=47.8 K 2024-11-16T04:24:37,812 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,812 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/5a34102eb2eb4472999d939e915b87f6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=181.0 K 2024-11-16T04:24:37,812 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,812 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/5a34102eb2eb4472999d939e915b87f6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e] 2024-11-16T04:24:37,814 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c5196a1480fa42f896556ad0cf27e773, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731731074055 2024-11-16T04:24:37,815 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a34102eb2eb4472999d939e915b87f6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731731074055 2024-11-16T04:24:37,815 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c59b88f3ca42689550878295c2b257, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731731075234 2024-11-16T04:24:37,816 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c105fd14ebdd4949914d19a1c504bf0d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731731075234 2024-11-16T04:24:37,817 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c8dae0aa35094e88876603d2e8977f33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731731075360 2024-11-16T04:24:37,824 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 110b1a4114024ae49fdaa197b2dbe06f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731077594 2024-11-16T04:24:37,824 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbc90b6e839e4861bc3e3aff87278f42, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731731075360 2024-11-16T04:24:37,825 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc6bc40000542eebfa0eb9b7a4cdb4e, keycount=450, bloomtype=ROW, size=81.1 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731077567 2024-11-16T04:24:37,841 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:37,844 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:37,845 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/945df78f401742bca4c0be00602fdf72 is 50, key is test_row_0/B:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,857 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111671536363a6f44f2eb10789fa219c3ab9_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:37,861 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111671536363a6f44f2eb10789fa219c3ab9_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:37,861 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111671536363a6f44f2eb10789fa219c3ab9_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742010_1186 (size=12629) 2024-11-16T04:24:37,891 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/945df78f401742bca4c0be00602fdf72 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/945df78f401742bca4c0be00602fdf72 2024-11-16T04:24:37,899 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into 945df78f401742bca4c0be00602fdf72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:37,899 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:37,899 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=12, startTime=1731731077808; duration=0sec 2024-11-16T04:24:37,900 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:37,900 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:37,900 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:37,902 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:37,902 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:37,902 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:37,902 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b223720d136747579c3a03edd77e6502, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/968dafcec3e74c588e3730fb315d6d00, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/a588a3eacbb449889b400410800b2c43, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/fd04c994e62d4e51b453b8bc2b2ae187] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=47.8 K 2024-11-16T04:24:37,903 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b223720d136747579c3a03edd77e6502, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731731074055 2024-11-16T04:24:37,903 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 968dafcec3e74c588e3730fb315d6d00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731731075234 2024-11-16T04:24:37,903 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a588a3eacbb449889b400410800b2c43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731731075360 2024-11-16T04:24:37,904 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting fd04c994e62d4e51b453b8bc2b2ae187, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731077594 2024-11-16T04:24:37,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742011_1187 (size=4469) 2024-11-16T04:24:37,910 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#156 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:37,911 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/90cd3f0e57964cffb1987efd692015c5 is 175, key is test_row_0/A:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,928 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:37,929 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/1bef56787e464a199b036b6d247ef568 is 50, key is test_row_0/C:col10/1731731077609/Put/seqid=0 2024-11-16T04:24:37,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742012_1188 (size=31583) 2024-11-16T04:24:37,942 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/90cd3f0e57964cffb1987efd692015c5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/90cd3f0e57964cffb1987efd692015c5 2024-11-16T04:24:37,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742013_1189 (size=12629) 2024-11-16T04:24:37,952 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into 90cd3f0e57964cffb1987efd692015c5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:37,953 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:37,953 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=12, startTime=1731731077808; duration=0sec 2024-11-16T04:24:37,953 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/1bef56787e464a199b036b6d247ef568 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/1bef56787e464a199b036b6d247ef568 2024-11-16T04:24:37,953 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:37,953 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:37,961 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into 1bef56787e464a199b036b6d247ef568(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:37,961 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:37,961 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=12, startTime=1731731077808; duration=0sec 2024-11-16T04:24:37,962 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:37,962 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:37,985 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:37,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T04:24:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731138003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731138003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731138004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160c5d76daf8084baa820a35ae81324879_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731077658/Put/seqid=0 2024-11-16T04:24:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731138012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731138012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742014_1190 (size=12304) 2024-11-16T04:24:38,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:38,045 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160c5d76daf8084baa820a35ae81324879_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160c5d76daf8084baa820a35ae81324879_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:38,046 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/3f8be357f7d54f1b869011d3b5aa1cd5, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:38,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/3f8be357f7d54f1b869011d3b5aa1cd5 is 175, key is test_row_0/A:col10/1731731077658/Put/seqid=0 2024-11-16T04:24:38,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742015_1191 (size=31105) 2024-11-16T04:24:38,061 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=228, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/3f8be357f7d54f1b869011d3b5aa1cd5 2024-11-16T04:24:38,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/e85b7f9f1d2e4c7db962a63725689f2c is 50, key is test_row_0/B:col10/1731731077658/Put/seqid=0 2024-11-16T04:24:38,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742016_1192 (size=12151) 2024-11-16T04:24:38,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731138114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731138123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731138123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731138123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731138123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731138323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731138333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731138333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731138334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731138334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/e85b7f9f1d2e4c7db962a63725689f2c 2024-11-16T04:24:38,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/010b0a9a21954dbfb1950f205d922562 is 50, key is test_row_0/C:col10/1731731077658/Put/seqid=0 2024-11-16T04:24:38,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742017_1193 (size=12151) 2024-11-16T04:24:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731138633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731138643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731138643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731138643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731138643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:38,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:38,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/010b0a9a21954dbfb1950f205d922562 2024-11-16T04:24:38,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/3f8be357f7d54f1b869011d3b5aa1cd5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5 2024-11-16T04:24:38,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5, entries=150, sequenceid=228, filesize=30.4 K 2024-11-16T04:24:38,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/e85b7f9f1d2e4c7db962a63725689f2c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/e85b7f9f1d2e4c7db962a63725689f2c 2024-11-16T04:24:38,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/e85b7f9f1d2e4c7db962a63725689f2c, entries=150, sequenceid=228, filesize=11.9 K 2024-11-16T04:24:38,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/010b0a9a21954dbfb1950f205d922562 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/010b0a9a21954dbfb1950f205d922562 2024-11-16T04:24:38,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/010b0a9a21954dbfb1950f205d922562, entries=150, sequenceid=228, filesize=11.9 K 2024-11-16T04:24:38,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for add53441e0ab2ad8f2f1f109ed502c64 in 943ms, sequenceid=228, compaction requested=false 2024-11-16T04:24:38,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:38,986 INFO [master/fda3efb342d8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T04:24:38,986 INFO [master/fda3efb342d8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T04:24:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:39,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:24:39,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111678af183b092f4b0582f6483819317089_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:39,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742018_1194 (size=14794) 2024-11-16T04:24:39,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:39,166 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111678af183b092f4b0582f6483819317089_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111678af183b092f4b0582f6483819317089_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:39,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/429cca0ef5ea4059a2555e37c8432f22, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:39,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/429cca0ef5ea4059a2555e37c8432f22 is 175, key is test_row_0/A:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:39,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742019_1195 (size=39749) 2024-11-16T04:24:39,177 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/429cca0ef5ea4059a2555e37c8432f22 2024-11-16T04:24:39,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/938cf9944bd84080a891f02efdc4bfd9 is 50, key is test_row_0/B:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:39,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731139180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731139180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731139188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731139190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731139192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742020_1196 (size=12151) 2024-11-16T04:24:39,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731139293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731139293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731139304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731139304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731139304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T04:24:39,323 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:39,324 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T04:24:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T04:24:39,326 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:39,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:39,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T04:24:39,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T04:24:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731139505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731139505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731139514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731139514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731139514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/938cf9944bd84080a891f02efdc4bfd9 2024-11-16T04:24:39,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/867f3afb205d4cdabc328248966c03e5 is 50, key is test_row_0/C:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:39,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742021_1197 (size=12151) 2024-11-16T04:24:39,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T04:24:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T04:24:39,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T04:24:39,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:39,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731139814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731139814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731139823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731139823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731139823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:39,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T04:24:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:39,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T04:24:40,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/867f3afb205d4cdabc328248966c03e5 2024-11-16T04:24:40,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/429cca0ef5ea4059a2555e37c8432f22 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22 2024-11-16T04:24:40,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22, entries=200, sequenceid=239, filesize=38.8 K 2024-11-16T04:24:40,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/938cf9944bd84080a891f02efdc4bfd9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/938cf9944bd84080a891f02efdc4bfd9 2024-11-16T04:24:40,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/938cf9944bd84080a891f02efdc4bfd9, entries=150, sequenceid=239, filesize=11.9 K 2024-11-16T04:24:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/867f3afb205d4cdabc328248966c03e5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/867f3afb205d4cdabc328248966c03e5 2024-11-16T04:24:40,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/867f3afb205d4cdabc328248966c03e5, entries=150, sequenceid=239, filesize=11.9 K 2024-11-16T04:24:40,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for add53441e0ab2ad8f2f1f109ed502c64 in 900ms, sequenceid=239, compaction requested=true 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:40,046 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:40,046 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:40,047 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:40,047 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:40,047 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:40,047 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:40,047 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:40,047 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:40,047 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/90cd3f0e57964cffb1987efd692015c5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=100.0 K 2024-11-16T04:24:40,047 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/945df78f401742bca4c0be00602fdf72, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/e85b7f9f1d2e4c7db962a63725689f2c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/938cf9944bd84080a891f02efdc4bfd9] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=36.1 K 2024-11-16T04:24:40,047 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:40,047 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/90cd3f0e57964cffb1987efd692015c5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22] 2024-11-16T04:24:40,047 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 945df78f401742bca4c0be00602fdf72, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731077594 2024-11-16T04:24:40,048 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90cd3f0e57964cffb1987efd692015c5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731077594 2024-11-16T04:24:40,048 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting e85b7f9f1d2e4c7db962a63725689f2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731731077658 2024-11-16T04:24:40,048 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f8be357f7d54f1b869011d3b5aa1cd5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731731077658 2024-11-16T04:24:40,048 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 938cf9944bd84080a891f02efdc4bfd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731731078000 2024-11-16T04:24:40,048 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429cca0ef5ea4059a2555e37c8432f22, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731731078000 2024-11-16T04:24:40,056 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:40,057 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/34d42812ab5b4a2c91ffa082f3969390 is 50, key is test_row_0/B:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:40,059 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:40,061 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116df9a65e5e29e453da345fb16ee6cb5b5_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:40,063 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116df9a65e5e29e453da345fb16ee6cb5b5_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:40,063 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116df9a65e5e29e453da345fb16ee6cb5b5_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742022_1198 (size=12731) 2024-11-16T04:24:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742023_1199 (size=4469) 2024-11-16T04:24:40,081 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#166 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:40,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/fbc767f395644dce851c6b977ac1b272 is 175, key is test_row_0/A:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:40,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T04:24:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:40,093 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:24:40,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742024_1200 (size=31685) 2024-11-16T04:24:40,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1f3d360e9bf4a27b68a334bc454ffd1_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731079189/Put/seqid=0 2024-11-16T04:24:40,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742025_1201 (size=12454) 2024-11-16T04:24:40,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:40,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b1f3d360e9bf4a27b68a334bc454ffd1_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1f3d360e9bf4a27b68a334bc454ffd1_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/b40fbffe4f3c44a5ba6472620bf61ecc, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:40,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/b40fbffe4f3c44a5ba6472620bf61ecc is 175, key is test_row_0/A:col10/1731731079189/Put/seqid=0 2024-11-16T04:24:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742026_1202 (size=31255) 2024-11-16T04:24:40,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731140338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731140338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731140338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731140343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731140343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731140454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731140454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731140455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T04:24:40,490 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/34d42812ab5b4a2c91ffa082f3969390 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/34d42812ab5b4a2c91ffa082f3969390 2024-11-16T04:24:40,498 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into 34d42812ab5b4a2c91ffa082f3969390(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:40,498 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:40,498 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=13, startTime=1731731080046; duration=0sec 2024-11-16T04:24:40,498 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:40,498 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:40,498 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:40,500 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:40,500 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:40,501 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:40,501 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/1bef56787e464a199b036b6d247ef568, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/010b0a9a21954dbfb1950f205d922562, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/867f3afb205d4cdabc328248966c03e5] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=36.1 K 2024-11-16T04:24:40,502 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bef56787e464a199b036b6d247ef568, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731077594 2024-11-16T04:24:40,502 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 010b0a9a21954dbfb1950f205d922562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1731731077658 2024-11-16T04:24:40,503 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 867f3afb205d4cdabc328248966c03e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731731078000 2024-11-16T04:24:40,504 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/fbc767f395644dce851c6b977ac1b272 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc767f395644dce851c6b977ac1b272 2024-11-16T04:24:40,511 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into fbc767f395644dce851c6b977ac1b272(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:40,511 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:40,511 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=13, startTime=1731731080046; duration=0sec 2024-11-16T04:24:40,511 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:40,511 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:40,514 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:40,514 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/cf7024e248104098a23150b795601703 is 50, key is test_row_0/C:col10/1731731079144/Put/seqid=0 2024-11-16T04:24:40,521 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/b40fbffe4f3c44a5ba6472620bf61ecc 2024-11-16T04:24:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/80654ca327d448699ec44cd54b36c375 is 50, key is test_row_0/B:col10/1731731079189/Put/seqid=0 2024-11-16T04:24:40,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742027_1203 (size=12731) 2024-11-16T04:24:40,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742028_1204 (size=12301) 2024-11-16T04:24:40,558 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/80654ca327d448699ec44cd54b36c375 2024-11-16T04:24:40,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/d94f339d91b24b4f8fd568875a7c5701 is 50, key is test_row_0/C:col10/1731731079189/Put/seqid=0 2024-11-16T04:24:40,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742029_1205 (size=12301) 2024-11-16T04:24:40,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731140664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731140664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731140664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,959 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/cf7024e248104098a23150b795601703 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/cf7024e248104098a23150b795601703 2024-11-16T04:24:40,964 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into cf7024e248104098a23150b795601703(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:40,964 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:40,964 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=13, startTime=1731731080046; duration=0sec 2024-11-16T04:24:40,964 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:40,965 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:40,974 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/d94f339d91b24b4f8fd568875a7c5701 2024-11-16T04:24:40,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731140973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731140973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731140973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:40,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/b40fbffe4f3c44a5ba6472620bf61ecc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc 2024-11-16T04:24:40,985 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc, entries=150, sequenceid=264, filesize=30.5 K 2024-11-16T04:24:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/80654ca327d448699ec44cd54b36c375 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/80654ca327d448699ec44cd54b36c375 2024-11-16T04:24:40,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/80654ca327d448699ec44cd54b36c375, entries=150, sequenceid=264, filesize=12.0 K 2024-11-16T04:24:40,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/d94f339d91b24b4f8fd568875a7c5701 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d94f339d91b24b4f8fd568875a7c5701 2024-11-16T04:24:40,999 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d94f339d91b24b4f8fd568875a7c5701, entries=150, sequenceid=264, filesize=12.0 K 2024-11-16T04:24:41,000 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for add53441e0ab2ad8f2f1f109ed502c64 in 907ms, sequenceid=264, compaction requested=false 2024-11-16T04:24:41,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:41,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T04:24:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T04:24:41,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T04:24:41,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6750 sec 2024-11-16T04:24:41,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6800 sec 2024-11-16T04:24:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:41,355 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:41,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:24:41,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b570bbc04a86479c88c6de699d83b7ac_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:41,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742030_1206 (size=12454) 2024-11-16T04:24:41,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:41,380 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b570bbc04a86479c88c6de699d83b7ac_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b570bbc04a86479c88c6de699d83b7ac_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:41,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/aea485207fa44dccaf60ff23ca77c8fb, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:41,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/aea485207fa44dccaf60ff23ca77c8fb is 175, key is test_row_0/A:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:41,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742031_1207 (size=31255) 2024-11-16T04:24:41,390 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/aea485207fa44dccaf60ff23ca77c8fb 2024-11-16T04:24:41,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/1c4f84cc079f4ba79161ba1b0e31b6ea is 50, key is test_row_0/B:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731141403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731141402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742032_1208 (size=12301) 2024-11-16T04:24:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T04:24:41,474 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:41,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T04:24:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:41,475 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:41,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:41,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:41,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731141483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731141484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731141484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731141513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731141513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:41,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T04:24:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731141723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731141723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:41,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:41,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T04:24:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:41,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/1c4f84cc079f4ba79161ba1b0e31b6ea 2024-11-16T04:24:41,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/8fd7ce4042c24e4c816802bd2fde74cd is 50, key is test_row_0/C:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:41,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742033_1209 (size=12301) 2024-11-16T04:24:41,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T04:24:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:41,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731142033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731142034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T04:24:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:42,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/8fd7ce4042c24e4c816802bd2fde74cd 2024-11-16T04:24:42,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/aea485207fa44dccaf60ff23ca77c8fb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb 2024-11-16T04:24:42,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb, entries=150, sequenceid=279, filesize=30.5 K 2024-11-16T04:24:42,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/1c4f84cc079f4ba79161ba1b0e31b6ea as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/1c4f84cc079f4ba79161ba1b0e31b6ea 2024-11-16T04:24:42,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/1c4f84cc079f4ba79161ba1b0e31b6ea, entries=150, sequenceid=279, filesize=12.0 K 2024-11-16T04:24:42,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/8fd7ce4042c24e4c816802bd2fde74cd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8fd7ce4042c24e4c816802bd2fde74cd 2024-11-16T04:24:42,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T04:24:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:24:42,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8fd7ce4042c24e4c816802bd2fde74cd, entries=150, sequenceid=279, filesize=12.0 K 2024-11-16T04:24:42,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for add53441e0ab2ad8f2f1f109ed502c64 in 887ms, sequenceid=279, compaction requested=true 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:42,242 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:42,242 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:42,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:42,243 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:42,243 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:42,243 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:42,243 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:42,243 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,243 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,243 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc767f395644dce851c6b977ac1b272, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=92.0 K 2024-11-16T04:24:42,243 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/34d42812ab5b4a2c91ffa082f3969390, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/80654ca327d448699ec44cd54b36c375, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/1c4f84cc079f4ba79161ba1b0e31b6ea] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=36.5 K 2024-11-16T04:24:42,243 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,243 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc767f395644dce851c6b977ac1b272, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb] 2024-11-16T04:24:42,244 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 34d42812ab5b4a2c91ffa082f3969390, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731731078000 2024-11-16T04:24:42,244 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbc767f395644dce851c6b977ac1b272, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731731078000 2024-11-16T04:24:42,244 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 80654ca327d448699ec44cd54b36c375, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731731079173 2024-11-16T04:24:42,244 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40fbffe4f3c44a5ba6472620bf61ecc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731731079173 2024-11-16T04:24:42,244 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c4f84cc079f4ba79161ba1b0e31b6ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731731080325 2024-11-16T04:24:42,244 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting aea485207fa44dccaf60ff23ca77c8fb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731731080325 2024-11-16T04:24:42,249 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:42,252 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:42,252 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/fb93a817cb31404dba4bd079d4c8776e is 50, key is test_row_0/B:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:42,254 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411166ec32f52426b46a096ca0307a56e7448_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:42,256 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411166ec32f52426b46a096ca0307a56e7448_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:42,256 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166ec32f52426b46a096ca0307a56e7448_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:42,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742034_1210 (size=12983) 2024-11-16T04:24:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742035_1211 (size=4469) 2024-11-16T04:24:42,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T04:24:42,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,392 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:24:42,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165897faaa8e0d42d3b24a5d8768260315_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731081401/Put/seqid=0 2024-11-16T04:24:42,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742036_1212 (size=12454) 2024-11-16T04:24:42,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:42,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:42,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731142509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731142510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731142511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731142544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731142544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,549 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:42,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731142613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731142623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731142623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,667 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/fb93a817cb31404dba4bd079d4c8776e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/fb93a817cb31404dba4bd079d4c8776e 2024-11-16T04:24:42,672 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into fb93a817cb31404dba4bd079d4c8776e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:42,672 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:42,672 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=13, startTime=1731731082242; duration=0sec 2024-11-16T04:24:42,673 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:42,673 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:42,673 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:42,674 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:42,674 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:42,674 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:42,674 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/cf7024e248104098a23150b795601703, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d94f339d91b24b4f8fd568875a7c5701, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8fd7ce4042c24e4c816802bd2fde74cd] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=36.5 K 2024-11-16T04:24:42,674 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#174 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:42,674 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting cf7024e248104098a23150b795601703, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731731078000 2024-11-16T04:24:42,675 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/0df48bfc73f348e0b9f870b72c717506 is 175, key is test_row_0/A:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:42,675 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d94f339d91b24b4f8fd568875a7c5701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731731079173 2024-11-16T04:24:42,675 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd7ce4042c24e4c816802bd2fde74cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731731080325 2024-11-16T04:24:42,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742037_1213 (size=31937) 2024-11-16T04:24:42,686 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:42,687 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/4e789cfbcb07460790532bc9372228ba is 50, key is test_row_0/C:col10/1731731080325/Put/seqid=0 2024-11-16T04:24:42,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742038_1214 (size=12983) 2024-11-16T04:24:42,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:42,808 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165897faaa8e0d42d3b24a5d8768260315_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165897faaa8e0d42d3b24a5d8768260315_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:42,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/ccc19271c73d4f879a9e87d731c4ef99, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:42,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/ccc19271c73d4f879a9e87d731c4ef99 is 175, key is test_row_0/A:col10/1731731081401/Put/seqid=0 2024-11-16T04:24:42,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742039_1215 (size=31255) 2024-11-16T04:24:42,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731142823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731142834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:42,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731142834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:42,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,084 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/0df48bfc73f348e0b9f870b72c717506 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0df48bfc73f348e0b9f870b72c717506 2024-11-16T04:24:43,089 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into 0df48bfc73f348e0b9f870b72c717506(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:43,089 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:43,089 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=13, startTime=1731731082242; duration=0sec 2024-11-16T04:24:43,089 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:43,089 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:43,094 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/4e789cfbcb07460790532bc9372228ba as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/4e789cfbcb07460790532bc9372228ba 2024-11-16T04:24:43,101 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into 4e789cfbcb07460790532bc9372228ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:43,101 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:43,101 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=13, startTime=1731731082242; duration=0sec 2024-11-16T04:24:43,101 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:43,101 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:43,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731143133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731143143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731143143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,214 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/ccc19271c73d4f879a9e87d731c4ef99 2024-11-16T04:24:43,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/6421724e551d4560acfb00ffcb8bbce9 is 50, key is test_row_0/B:col10/1731731081401/Put/seqid=0 2024-11-16T04:24:43,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742040_1216 (size=12301) 2024-11-16T04:24:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731143553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,554 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in add53441e0ab2ad8f2f1f109ed502c64 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731143563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,564 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in add53441e0ab2ad8f2f1f109ed502c64 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:43,627 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/6421724e551d4560acfb00ffcb8bbce9 2024-11-16T04:24:43,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/6fccb4683e96492a8b275ee7155523dd is 50, key is test_row_0/C:col10/1731731081401/Put/seqid=0 2024-11-16T04:24:43,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742041_1217 (size=12301) 2024-11-16T04:24:43,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731143643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731143653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731143653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:44,074 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/6fccb4683e96492a8b275ee7155523dd 2024-11-16T04:24:44,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/ccc19271c73d4f879a9e87d731c4ef99 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99 2024-11-16T04:24:44,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99, entries=150, sequenceid=303, filesize=30.5 K 2024-11-16T04:24:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/6421724e551d4560acfb00ffcb8bbce9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/6421724e551d4560acfb00ffcb8bbce9 2024-11-16T04:24:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,088 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/6421724e551d4560acfb00ffcb8bbce9, entries=150, sequenceid=303, filesize=12.0 K 2024-11-16T04:24:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/6fccb4683e96492a8b275ee7155523dd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/6fccb4683e96492a8b275ee7155523dd 2024-11-16T04:24:44,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/6fccb4683e96492a8b275ee7155523dd, entries=150, sequenceid=303, filesize=12.0 K 2024-11-16T04:24:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,095 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for add53441e0ab2ad8f2f1f109ed502c64 in 1703ms, sequenceid=303, compaction requested=false 2024-11-16T04:24:44,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:44,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T04:24:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T04:24:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6210 sec 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.6260 sec 2024-11-16T04:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:44,671 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:44,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:24:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165beb914436754ea8a78a6dccf07be552_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742042_1218 (size=12454) 2024-11-16T04:24:44,740 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:44,748 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165beb914436754ea8a78a6dccf07be552_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165beb914436754ea8a78a6dccf07be552_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731144741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731144742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:44,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731144743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:44,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c21dbffb79be45698517dfc9e9ef71b9, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:44,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c21dbffb79be45698517dfc9e9ef71b9 is 175, key is test_row_0/A:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742043_1219 (size=31251) 2024-11-16T04:24:44,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731144854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:44,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731144854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:44,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:44,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731144855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:44,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731145064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731145064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731145064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,179 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=319, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c21dbffb79be45698517dfc9e9ef71b9 2024-11-16T04:24:45,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/37c84e42440543919759f0bc7b1ad6a0 is 50, key is test_row_0/B:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:45,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742044_1220 (size=9857) 2024-11-16T04:24:45,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/37c84e42440543919759f0bc7b1ad6a0 2024-11-16T04:24:45,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/7bf782ce1e8846c6bd4aa2ee55834ebf is 50, key is test_row_0/C:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:45,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742045_1221 (size=9857) 2024-11-16T04:24:45,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731145373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731145373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731145374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731145573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731145573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,575 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in add53441e0ab2ad8f2f1f109ed502c64 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:45,576 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in add53441e0ab2ad8f2f1f109ed502c64 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/7bf782ce1e8846c6bd4aa2ee55834ebf 2024-11-16T04:24:45,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c21dbffb79be45698517dfc9e9ef71b9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9 2024-11-16T04:24:45,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9, entries=150, sequenceid=319, filesize=30.5 K 2024-11-16T04:24:45,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/37c84e42440543919759f0bc7b1ad6a0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/37c84e42440543919759f0bc7b1ad6a0 2024-11-16T04:24:45,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/37c84e42440543919759f0bc7b1ad6a0, entries=100, sequenceid=319, filesize=9.6 K 2024-11-16T04:24:45,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/7bf782ce1e8846c6bd4aa2ee55834ebf as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7bf782ce1e8846c6bd4aa2ee55834ebf 2024-11-16T04:24:45,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7bf782ce1e8846c6bd4aa2ee55834ebf, entries=100, sequenceid=319, filesize=9.6 K 2024-11-16T04:24:45,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for add53441e0ab2ad8f2f1f109ed502c64 in 959ms, sequenceid=319, compaction requested=true 2024-11-16T04:24:45,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:45,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:45,630 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:45,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:45,631 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:45,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:45,632 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:45,632 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:45,632 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:45,632 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/fb93a817cb31404dba4bd079d4c8776e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/6421724e551d4560acfb00ffcb8bbce9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/37c84e42440543919759f0bc7b1ad6a0] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=34.3 K 2024-11-16T04:24:45,633 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:45,633 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:45,633 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:45,633 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting fb93a817cb31404dba4bd079d4c8776e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731731080325 2024-11-16T04:24:45,633 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0df48bfc73f348e0b9f870b72c717506, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=92.2 K 2024-11-16T04:24:45,633 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:45,633 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0df48bfc73f348e0b9f870b72c717506, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9] 2024-11-16T04:24:45,633 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6421724e551d4560acfb00ffcb8bbce9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731081394 2024-11-16T04:24:45,633 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df48bfc73f348e0b9f870b72c717506, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731731080325 2024-11-16T04:24:45,634 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 37c84e42440543919759f0bc7b1ad6a0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731731082509 2024-11-16T04:24:45,634 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc19271c73d4f879a9e87d731c4ef99, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731081394 2024-11-16T04:24:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T04:24:45,634 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c21dbffb79be45698517dfc9e9ef71b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731731082508 2024-11-16T04:24:45,634 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:45,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T04:24:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T04:24:45,637 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:45,637 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:45,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:45,646 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:45,647 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/3ddc16798b7e4a0389b841ffa299220d is 50, key is test_row_0/B:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:45,648 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:45,657 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163ba589bff490485cae8f06608e9a4743_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:45,659 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163ba589bff490485cae8f06608e9a4743_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:45,659 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163ba589bff490485cae8f06608e9a4743_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742046_1222 (size=13085) 2024-11-16T04:24:45,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742047_1223 (size=4469) 2024-11-16T04:24:45,680 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/3ddc16798b7e4a0389b841ffa299220d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/3ddc16798b7e4a0389b841ffa299220d 2024-11-16T04:24:45,689 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into 3ddc16798b7e4a0389b841ffa299220d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:45,689 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:45,689 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=13, startTime=1731731085631; duration=0sec 2024-11-16T04:24:45,689 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:45,689 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:45,689 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:24:45,690 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:24:45,690 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:45,690 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:45,690 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/4e789cfbcb07460790532bc9372228ba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/6fccb4683e96492a8b275ee7155523dd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7bf782ce1e8846c6bd4aa2ee55834ebf] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=34.3 K 2024-11-16T04:24:45,691 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e789cfbcb07460790532bc9372228ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731731080325 2024-11-16T04:24:45,691 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fccb4683e96492a8b275ee7155523dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731731081394 2024-11-16T04:24:45,692 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf782ce1e8846c6bd4aa2ee55834ebf, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731731082509 2024-11-16T04:24:45,704 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:45,705 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/eca3db9224ba4bdf969828882066c165 is 50, key is test_row_0/C:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:45,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742048_1224 (size=13085) 2024-11-16T04:24:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T04:24:45,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T04:24:45,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:24:45,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116853cc34687344e12a2d3074b6594ba78_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731084741/Put/seqid=0 2024-11-16T04:24:45,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742049_1225 (size=12454) 2024-11-16T04:24:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:45,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731145901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731145901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:45,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731145902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:45,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T04:24:46,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731146014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731146014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731146014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,076 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:46,077 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/3052cb9b6ff64fd1a69bfba03a624bea is 175, key is test_row_0/A:col10/1731731082509/Put/seqid=0 2024-11-16T04:24:46,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742050_1226 (size=32146) 2024-11-16T04:24:46,122 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/eca3db9224ba4bdf969828882066c165 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/eca3db9224ba4bdf969828882066c165 2024-11-16T04:24:46,127 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into eca3db9224ba4bdf969828882066c165(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:46,127 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:46,127 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=13, startTime=1731731085631; duration=0sec 2024-11-16T04:24:46,128 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:46,128 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:46,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:46,209 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116853cc34687344e12a2d3074b6594ba78_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116853cc34687344e12a2d3074b6594ba78_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:46,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/eb92a82e2fab49758c8107559e862fa6, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:46,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/eb92a82e2fab49758c8107559e862fa6 is 175, key is test_row_0/A:col10/1731731084741/Put/seqid=0 2024-11-16T04:24:46,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742051_1227 (size=31255) 2024-11-16T04:24:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731146223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731146223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731146223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T04:24:46,488 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/3052cb9b6ff64fd1a69bfba03a624bea as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3052cb9b6ff64fd1a69bfba03a624bea 2024-11-16T04:24:46,495 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into 3052cb9b6ff64fd1a69bfba03a624bea(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:46,495 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:46,495 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=13, startTime=1731731085630; duration=0sec 2024-11-16T04:24:46,495 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:46,495 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:46,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731146533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731146533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731146533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:46,614 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/eb92a82e2fab49758c8107559e862fa6 2024-11-16T04:24:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/ad62c1e94cdb4b6f90cc8b76d06658b4 is 50, key is test_row_0/B:col10/1731731084741/Put/seqid=0 2024-11-16T04:24:46,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742052_1228 (size=12301) 2024-11-16T04:24:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T04:24:47,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/ad62c1e94cdb4b6f90cc8b76d06658b4 2024-11-16T04:24:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/36cdae2c85cf4b0aa9ae456a392c3085 is 50, key is test_row_0/C:col10/1731731084741/Put/seqid=0 2024-11-16T04:24:47,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742053_1229 (size=12301) 2024-11-16T04:24:47,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/36cdae2c85cf4b0aa9ae456a392c3085 2024-11-16T04:24:47,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/eb92a82e2fab49758c8107559e862fa6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6 2024-11-16T04:24:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731147043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:47,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:47,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731147043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:47,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:47,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731147043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:47,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:47,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6, entries=150, sequenceid=343, filesize=30.5 K 2024-11-16T04:24:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/ad62c1e94cdb4b6f90cc8b76d06658b4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/ad62c1e94cdb4b6f90cc8b76d06658b4 2024-11-16T04:24:47,059 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/ad62c1e94cdb4b6f90cc8b76d06658b4, entries=150, sequenceid=343, filesize=12.0 K 2024-11-16T04:24:47,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/36cdae2c85cf4b0aa9ae456a392c3085 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36cdae2c85cf4b0aa9ae456a392c3085 2024-11-16T04:24:47,068 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36cdae2c85cf4b0aa9ae456a392c3085, entries=150, sequenceid=343, filesize=12.0 K 2024-11-16T04:24:47,069 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for add53441e0ab2ad8f2f1f109ed502c64 in 1278ms, sequenceid=343, compaction requested=false 2024-11-16T04:24:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:47,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T04:24:47,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T04:24:47,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T04:24:47,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-16T04:24:47,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.4390 sec 2024-11-16T04:24:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T04:24:47,784 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:47,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:24:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-16T04:24:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T04:24:47,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:24:47,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:24:47,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:24:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T04:24:47,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T04:24:47,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:47,939 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:24:47,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166b5d871dc1f245b786cc17841da04be5_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731085900/Put/seqid=0 2024-11-16T04:24:47,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742054_1230 (size=12454) 2024-11-16T04:24:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:48,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. as already flushing 2024-11-16T04:24:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731148096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731148096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731148097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T04:24:48,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731148203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731148204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731148204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:48,383 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166b5d871dc1f245b786cc17841da04be5_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166b5d871dc1f245b786cc17841da04be5_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:48,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/23d9afb9d3da49e5ab00b282bfc2a9b3, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:48,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/23d9afb9d3da49e5ab00b282bfc2a9b3 is 175, key is test_row_0/A:col10/1731731085900/Put/seqid=0 2024-11-16T04:24:48,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742055_1231 (size=31255) 2024-11-16T04:24:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T04:24:48,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731148414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731148414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731148414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731148724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731148724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731148724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:48,735 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-16T04:24:48,735 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:48,735 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,735 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:48,738 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-16T04:24:48,738 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:48,738 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,738 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:48,740 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-16T04:24:48,740 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:48,740 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,740 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,740 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:48,740 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-16T04:24:48,740 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:48,740 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,741 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:48,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:48,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/23d9afb9d3da49e5ab00b282bfc2a9b3 2024-11-16T04:24:48,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/9a989b6423da4c418e7d97f8cb315121 is 50, key is test_row_0/B:col10/1731731085900/Put/seqid=0 2024-11-16T04:24:48,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742056_1232 (size=12301) 2024-11-16T04:24:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T04:24:49,206 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/9a989b6423da4c418e7d97f8cb315121 2024-11-16T04:24:49,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/582d4f9ecb974753a5191af870c1ac40 is 50, key is test_row_0/C:col10/1731731085900/Put/seqid=0 2024-11-16T04:24:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742057_1233 (size=12301) 2024-11-16T04:24:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57796 deadline: 1731731149233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57822 deadline: 1731731149233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57836 deadline: 1731731149233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:49,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:24:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57750 deadline: 1731731149604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57790 deadline: 1731731149604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:49,605 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in add53441e0ab2ad8f2f1f109ed502c64 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:49,605 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in add53441e0ab2ad8f2f1f109ed502c64 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:24:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:24:49,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:24:49,616 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/582d4f9ecb974753a5191af870c1ac40 2024-11-16T04:24:49,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/23d9afb9d3da49e5ab00b282bfc2a9b3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3 2024-11-16T04:24:49,623 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3, entries=150, sequenceid=358, filesize=30.5 K 2024-11-16T04:24:49,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/9a989b6423da4c418e7d97f8cb315121 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/9a989b6423da4c418e7d97f8cb315121 2024-11-16T04:24:49,627 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/9a989b6423da4c418e7d97f8cb315121, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T04:24:49,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/582d4f9ecb974753a5191af870c1ac40 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/582d4f9ecb974753a5191af870c1ac40 2024-11-16T04:24:49,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/582d4f9ecb974753a5191af870c1ac40, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T04:24:49,634 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for add53441e0ab2ad8f2f1f109ed502c64 in 1696ms, sequenceid=358, compaction requested=true 2024-11-16T04:24:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-16T04:24:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-16T04:24:49,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T04:24:49,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-11-16T04:24:49,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8540 sec 2024-11-16T04:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T04:24:49,935 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:24:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:50,244 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:24:50,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:24:50,244 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-16T04:24:50,244 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-16T04:24:50,244 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:50,244 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:50,245 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:50,245 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:50,245 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:50,245 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:50,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:50,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:50,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111651cdb44880a547149d9700a27cc5e966_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_0/A:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:50,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742058_1234 (size=12454) 2024-11-16T04:24:50,254 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-16T04:24:50,255 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:50,255 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:50,255 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:50,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:50,654 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:24:50,659 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111651cdb44880a547149d9700a27cc5e966_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111651cdb44880a547149d9700a27cc5e966_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:50,660 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8e6ebd66448b47ca832017c15b5a5ff8, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:50,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8e6ebd66448b47ca832017c15b5a5ff8 is 175, key is test_row_0/A:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:50,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742059_1235 (size=31255) 2024-11-16T04:24:51,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8e6ebd66448b47ca832017c15b5a5ff8 2024-11-16T04:24:51,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/feadb24c1ef1470bb2a58bbc3d7d9766 is 50, key is test_row_0/B:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:51,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742060_1236 (size=12301) 2024-11-16T04:24:51,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/feadb24c1ef1470bb2a58bbc3d7d9766 2024-11-16T04:24:51,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/093704ba89a24b20a27964435271932e is 50, key is test_row_0/C:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:51,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742061_1237 (size=12301) 2024-11-16T04:24:51,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/093704ba89a24b20a27964435271932e 2024-11-16T04:24:51,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/8e6ebd66448b47ca832017c15b5a5ff8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8 2024-11-16T04:24:51,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8, entries=150, sequenceid=383, filesize=30.5 K 2024-11-16T04:24:51,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/feadb24c1ef1470bb2a58bbc3d7d9766 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/feadb24c1ef1470bb2a58bbc3d7d9766 2024-11-16T04:24:51,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/feadb24c1ef1470bb2a58bbc3d7d9766, entries=150, sequenceid=383, filesize=12.0 K 2024-11-16T04:24:51,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/093704ba89a24b20a27964435271932e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/093704ba89a24b20a27964435271932e 2024-11-16T04:24:51,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/093704ba89a24b20a27964435271932e, entries=150, sequenceid=383, filesize=12.0 K 2024-11-16T04:24:51,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=6.71 KB/6870 for add53441e0ab2ad8f2f1f109ed502c64 in 1279ms, sequenceid=383, compaction requested=true 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:24:51,523 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:51,523 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store add53441e0ab2ad8f2f1f109ed502c64:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:24:51,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:51,524 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125911 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:51,524 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:51,524 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/A is initiating minor compaction (all files) 2024-11-16T04:24:51,524 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/B is initiating minor compaction (all files) 2024-11-16T04:24:51,524 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/A in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:51,524 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/B in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:51,524 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3052cb9b6ff64fd1a69bfba03a624bea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=123.0 K 2024-11-16T04:24:51,524 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/3ddc16798b7e4a0389b841ffa299220d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/ad62c1e94cdb4b6f90cc8b76d06658b4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/9a989b6423da4c418e7d97f8cb315121, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/feadb24c1ef1470bb2a58bbc3d7d9766] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=48.8 K 2024-11-16T04:24:51,524 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:51,524 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3052cb9b6ff64fd1a69bfba03a624bea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8] 2024-11-16T04:24:51,525 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ddc16798b7e4a0389b841ffa299220d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731731081400 2024-11-16T04:24:51,525 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3052cb9b6ff64fd1a69bfba03a624bea, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731731081400 2024-11-16T04:24:51,525 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ad62c1e94cdb4b6f90cc8b76d06658b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731731084726 2024-11-16T04:24:51,525 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb92a82e2fab49758c8107559e862fa6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731731084726 2024-11-16T04:24:51,525 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a989b6423da4c418e7d97f8cb315121, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731731085898 2024-11-16T04:24:51,525 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23d9afb9d3da49e5ab00b282bfc2a9b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731731085898 2024-11-16T04:24:51,526 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting feadb24c1ef1470bb2a58bbc3d7d9766, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731731088095 2024-11-16T04:24:51,526 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e6ebd66448b47ca832017c15b5a5ff8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731731088095 2024-11-16T04:24:51,537 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#B#compaction#195 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:51,538 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/33d156bacb3e470f95b772aa5599b51b is 50, key is test_row_0/B:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:51,538 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:51,540 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411167a0dca4fec3e471faf1caa49e5a6231c_add53441e0ab2ad8f2f1f109ed502c64 store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:51,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742062_1238 (size=13221) 2024-11-16T04:24:51,569 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411167a0dca4fec3e471faf1caa49e5a6231c_add53441e0ab2ad8f2f1f109ed502c64, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:51,569 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167a0dca4fec3e471faf1caa49e5a6231c_add53441e0ab2ad8f2f1f109ed502c64 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:24:51,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742063_1239 (size=4469) 2024-11-16T04:24:51,950 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/33d156bacb3e470f95b772aa5599b51b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/33d156bacb3e470f95b772aa5599b51b 2024-11-16T04:24:51,956 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/B of add53441e0ab2ad8f2f1f109ed502c64 into 33d156bacb3e470f95b772aa5599b51b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:51,956 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:51,956 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/B, priority=12, startTime=1731731091523; duration=0sec 2024-11-16T04:24:51,956 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:24:51,957 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:B 2024-11-16T04:24:51,957 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:24:51,958 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:24:51,958 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): add53441e0ab2ad8f2f1f109ed502c64/C is initiating minor compaction (all files) 2024-11-16T04:24:51,958 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of add53441e0ab2ad8f2f1f109ed502c64/C in TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:51,958 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/eca3db9224ba4bdf969828882066c165, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36cdae2c85cf4b0aa9ae456a392c3085, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/582d4f9ecb974753a5191af870c1ac40, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/093704ba89a24b20a27964435271932e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp, totalSize=48.8 K 2024-11-16T04:24:51,958 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting eca3db9224ba4bdf969828882066c165, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731731081400 2024-11-16T04:24:51,958 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 36cdae2c85cf4b0aa9ae456a392c3085, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731731084726 2024-11-16T04:24:51,959 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 582d4f9ecb974753a5191af870c1ac40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731731085898 2024-11-16T04:24:51,959 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 093704ba89a24b20a27964435271932e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731731088095 2024-11-16T04:24:51,968 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#C#compaction#197 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:51,968 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/60b4318282a4450fad0d8eb4f50be79d is 50, key is test_row_0/C:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:51,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742064_1240 (size=13221) 2024-11-16T04:24:51,976 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): add53441e0ab2ad8f2f1f109ed502c64#A#compaction#196 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:24:51,976 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c9dcb5b720ec473db088f5ea7c7d9763 is 175, key is test_row_0/A:col10/1731731090243/Put/seqid=0 2024-11-16T04:24:51,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742065_1241 (size=32175) 2024-11-16T04:24:52,336 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T04:24:52,378 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/60b4318282a4450fad0d8eb4f50be79d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/60b4318282a4450fad0d8eb4f50be79d 2024-11-16T04:24:52,384 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/C of add53441e0ab2ad8f2f1f109ed502c64 into 60b4318282a4450fad0d8eb4f50be79d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:52,385 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:52,385 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/C, priority=12, startTime=1731731091523; duration=0sec 2024-11-16T04:24:52,385 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:52,385 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:C 2024-11-16T04:24:52,386 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/c9dcb5b720ec473db088f5ea7c7d9763 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c9dcb5b720ec473db088f5ea7c7d9763 2024-11-16T04:24:52,392 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in add53441e0ab2ad8f2f1f109ed502c64/A of add53441e0ab2ad8f2f1f109ed502c64 into c9dcb5b720ec473db088f5ea7c7d9763(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:24:52,392 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for add53441e0ab2ad8f2f1f109ed502c64: 2024-11-16T04:24:52,392 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., storeName=add53441e0ab2ad8f2f1f109ed502c64/A, priority=12, startTime=1731731091523; duration=0sec 2024-11-16T04:24:52,392 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:24:52,392 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: add53441e0ab2ad8f2f1f109ed502c64:A 2024-11-16T04:24:59,614 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-16T04:24:59,614 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:59,614 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:59,614 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:59,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:59,645 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-16T04:24:59,645 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:24:59,645 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:59,645 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:59,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3405 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3391 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1444 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4332 rows 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1442 2024-11-16T04:24:59,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4324 rows 2024-11-16T04:24:59,645 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:24:59,645 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:24:59,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:59,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:24:59,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:24:59,646 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T04:24:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T04:24:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T04:24:59,649 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731099649"}]},"ts":"1731731099649"} 2024-11-16T04:24:59,650 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T04:24:59,650 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T04:24:59,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:24:59,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, UNASSIGN}] 2024-11-16T04:24:59,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, UNASSIGN 2024-11-16T04:24:59,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:24:59,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, UNASSIGN because future has completed 2024-11-16T04:24:59,655 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:24:59,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:24:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T04:24:59,808 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:24:59,808 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:24:59,808 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing add53441e0ab2ad8f2f1f109ed502c64, disabling compactions & flushes 2024-11-16T04:24:59,808 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:59,808 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:59,808 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. after waiting 0 ms 2024-11-16T04:24:59,808 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:24:59,808 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing add53441e0ab2ad8f2f1f109ed502c64 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-16T04:24:59,816 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168f20a33ee8bb4c588ff0a1681feff37c_add53441e0ab2ad8f2f1f109ed502c64 is 50, key is test_row_1/A:col10/1731731099613/Put/seqid=0 2024-11-16T04:24:59,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742066_1242 (size=9914) 2024-11-16T04:24:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T04:25:00,228 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:00,233 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168f20a33ee8bb4c588ff0a1681feff37c_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168f20a33ee8bb4c588ff0a1681feff37c_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:00,235 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4520f38968b649ccbf4156cbdad83368, store: [table=TestAcidGuarantees family=A region=add53441e0ab2ad8f2f1f109ed502c64] 2024-11-16T04:25:00,235 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4520f38968b649ccbf4156cbdad83368 is 175, key is test_row_1/A:col10/1731731099613/Put/seqid=0 2024-11-16T04:25:00,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742067_1243 (size=22561) 2024-11-16T04:25:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T04:25:00,647 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4520f38968b649ccbf4156cbdad83368 2024-11-16T04:25:00,662 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/5f8ae7b0951148a4a572a7e338575d12 is 50, key is test_row_1/B:col10/1731731099613/Put/seqid=0 2024-11-16T04:25:00,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742068_1244 (size=9857) 2024-11-16T04:25:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T04:25:01,074 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/5f8ae7b0951148a4a572a7e338575d12 2024-11-16T04:25:01,081 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/8546ab1abee54e13b1a4151e5bd550e6 is 50, key is test_row_1/C:col10/1731731099613/Put/seqid=0 2024-11-16T04:25:01,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742069_1245 (size=9857) 2024-11-16T04:25:01,085 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/8546ab1abee54e13b1a4151e5bd550e6 2024-11-16T04:25:01,090 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/A/4520f38968b649ccbf4156cbdad83368 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4520f38968b649ccbf4156cbdad83368 2024-11-16T04:25:01,093 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4520f38968b649ccbf4156cbdad83368, entries=100, sequenceid=392, filesize=22.0 K 2024-11-16T04:25:01,094 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/B/5f8ae7b0951148a4a572a7e338575d12 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/5f8ae7b0951148a4a572a7e338575d12 2024-11-16T04:25:01,098 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/5f8ae7b0951148a4a572a7e338575d12, entries=100, sequenceid=392, filesize=9.6 K 2024-11-16T04:25:01,099 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/.tmp/C/8546ab1abee54e13b1a4151e5bd550e6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8546ab1abee54e13b1a4151e5bd550e6 2024-11-16T04:25:01,103 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8546ab1abee54e13b1a4151e5bd550e6, entries=100, sequenceid=392, filesize=9.6 K 2024-11-16T04:25:01,103 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for add53441e0ab2ad8f2f1f109ed502c64 in 1295ms, sequenceid=392, compaction requested=false 2024-11-16T04:25:01,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4b934f6f85f94ae7a4bbfa679a120efe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4858198f9a58408c8d1a8eb07300726c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/5a34102eb2eb4472999d939e915b87f6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/90cd3f0e57964cffb1987efd692015c5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc767f395644dce851c6b977ac1b272, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0df48bfc73f348e0b9f870b72c717506, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3052cb9b6ff64fd1a69bfba03a624bea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8] to archive 2024-11-16T04:25:01,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:01,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8593d11934f34b70851641f53616fcd9 2024-11-16T04:25:01,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/865b929b28354b918df2fa96c3a1d5a0 2024-11-16T04:25:01,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/acc9f002ffb2460c8e89a1fc54665fe0 2024-11-16T04:25:01,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/9c81c9b9a8434b268d47b7e79ad29334 2024-11-16T04:25:01,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4b934f6f85f94ae7a4bbfa679a120efe to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4b934f6f85f94ae7a4bbfa679a120efe 2024-11-16T04:25:01,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/449f49417b1d4118aa44be9816321ea4 2024-11-16T04:25:01,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4858198f9a58408c8d1a8eb07300726c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4858198f9a58408c8d1a8eb07300726c 2024-11-16T04:25:01,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/24626f0ca39d4fd4b16bd0974f23d98f 2024-11-16T04:25:01,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/407e9f6091be44d8b3168035593c6fd3 2024-11-16T04:25:01,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/2e78ecbdecf54f54a12fbea0cc95cb9a 2024-11-16T04:25:01,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/5a34102eb2eb4472999d939e915b87f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/5a34102eb2eb4472999d939e915b87f6 2024-11-16T04:25:01,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c105fd14ebdd4949914d19a1c504bf0d 2024-11-16T04:25:01,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc90b6e839e4861bc3e3aff87278f42 2024-11-16T04:25:01,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0bc6bc40000542eebfa0eb9b7a4cdb4e 2024-11-16T04:25:01,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/90cd3f0e57964cffb1987efd692015c5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/90cd3f0e57964cffb1987efd692015c5 2024-11-16T04:25:01,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3f8be357f7d54f1b869011d3b5aa1cd5 2024-11-16T04:25:01,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/429cca0ef5ea4059a2555e37c8432f22 2024-11-16T04:25:01,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc767f395644dce851c6b977ac1b272 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/fbc767f395644dce851c6b977ac1b272 2024-11-16T04:25:01,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/b40fbffe4f3c44a5ba6472620bf61ecc 2024-11-16T04:25:01,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0df48bfc73f348e0b9f870b72c717506 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/0df48bfc73f348e0b9f870b72c717506 2024-11-16T04:25:01,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/aea485207fa44dccaf60ff23ca77c8fb 2024-11-16T04:25:01,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/ccc19271c73d4f879a9e87d731c4ef99 2024-11-16T04:25:01,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3052cb9b6ff64fd1a69bfba03a624bea to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/3052cb9b6ff64fd1a69bfba03a624bea 2024-11-16T04:25:01,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c21dbffb79be45698517dfc9e9ef71b9 2024-11-16T04:25:01,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/eb92a82e2fab49758c8107559e862fa6 2024-11-16T04:25:01,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/23d9afb9d3da49e5ab00b282bfc2a9b3 2024-11-16T04:25:01,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/8e6ebd66448b47ca832017c15b5a5ff8 2024-11-16T04:25:01,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/8b81b374fa1543a7b1314e57186cd93d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a2e88bd4d35a4d489571ca045a6d9fba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/191ab5d50cdb4b51ae7d7fb7cd6b3146, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/71524e39404141978b78dfdaa2cf5d2c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b9072f1d99774f1182b8f9ee8a7ec75c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a8e141f8c67a4598a69843e2f769a00a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b18b25569b2342dca739bbab052b565c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/06ec234dcbf843e987ef8d35c94b1b0d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/f577ac913cff46069e58ab5a311b4dce, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c5196a1480fa42f896556ad0cf27e773, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/49b389bdc3a94f02a99d67dc371e5776, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/86c59b88f3ca42689550878295c2b257, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c8dae0aa35094e88876603d2e8977f33, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/945df78f401742bca4c0be00602fdf72, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/110b1a4114024ae49fdaa197b2dbe06f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/e85b7f9f1d2e4c7db962a63725689f2c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/34d42812ab5b4a2c91ffa082f3969390, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/938cf9944bd84080a891f02efdc4bfd9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/80654ca327d448699ec44cd54b36c375, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/fb93a817cb31404dba4bd079d4c8776e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/1c4f84cc079f4ba79161ba1b0e31b6ea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/6421724e551d4560acfb00ffcb8bbce9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/3ddc16798b7e4a0389b841ffa299220d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/37c84e42440543919759f0bc7b1ad6a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/ad62c1e94cdb4b6f90cc8b76d06658b4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/9a989b6423da4c418e7d97f8cb315121, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/feadb24c1ef1470bb2a58bbc3d7d9766] to archive 2024-11-16T04:25:01,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:01,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/8b81b374fa1543a7b1314e57186cd93d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/8b81b374fa1543a7b1314e57186cd93d 2024-11-16T04:25:01,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a2e88bd4d35a4d489571ca045a6d9fba to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a2e88bd4d35a4d489571ca045a6d9fba 2024-11-16T04:25:01,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/191ab5d50cdb4b51ae7d7fb7cd6b3146 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/191ab5d50cdb4b51ae7d7fb7cd6b3146 2024-11-16T04:25:01,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/71524e39404141978b78dfdaa2cf5d2c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/71524e39404141978b78dfdaa2cf5d2c 2024-11-16T04:25:01,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b9072f1d99774f1182b8f9ee8a7ec75c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b9072f1d99774f1182b8f9ee8a7ec75c 2024-11-16T04:25:01,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a8e141f8c67a4598a69843e2f769a00a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/a8e141f8c67a4598a69843e2f769a00a 2024-11-16T04:25:01,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b18b25569b2342dca739bbab052b565c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/b18b25569b2342dca739bbab052b565c 2024-11-16T04:25:01,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/06ec234dcbf843e987ef8d35c94b1b0d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/06ec234dcbf843e987ef8d35c94b1b0d 2024-11-16T04:25:01,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/f577ac913cff46069e58ab5a311b4dce to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/f577ac913cff46069e58ab5a311b4dce 2024-11-16T04:25:01,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c5196a1480fa42f896556ad0cf27e773 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c5196a1480fa42f896556ad0cf27e773 2024-11-16T04:25:01,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/49b389bdc3a94f02a99d67dc371e5776 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/49b389bdc3a94f02a99d67dc371e5776 2024-11-16T04:25:01,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/86c59b88f3ca42689550878295c2b257 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/86c59b88f3ca42689550878295c2b257 2024-11-16T04:25:01,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c8dae0aa35094e88876603d2e8977f33 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/c8dae0aa35094e88876603d2e8977f33 2024-11-16T04:25:01,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/945df78f401742bca4c0be00602fdf72 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/945df78f401742bca4c0be00602fdf72 2024-11-16T04:25:01,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/110b1a4114024ae49fdaa197b2dbe06f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/110b1a4114024ae49fdaa197b2dbe06f 2024-11-16T04:25:01,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/e85b7f9f1d2e4c7db962a63725689f2c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/e85b7f9f1d2e4c7db962a63725689f2c 2024-11-16T04:25:01,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/34d42812ab5b4a2c91ffa082f3969390 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/34d42812ab5b4a2c91ffa082f3969390 2024-11-16T04:25:01,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/938cf9944bd84080a891f02efdc4bfd9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/938cf9944bd84080a891f02efdc4bfd9 2024-11-16T04:25:01,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/80654ca327d448699ec44cd54b36c375 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/80654ca327d448699ec44cd54b36c375 2024-11-16T04:25:01,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/fb93a817cb31404dba4bd079d4c8776e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/fb93a817cb31404dba4bd079d4c8776e 2024-11-16T04:25:01,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/1c4f84cc079f4ba79161ba1b0e31b6ea to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/1c4f84cc079f4ba79161ba1b0e31b6ea 2024-11-16T04:25:01,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/6421724e551d4560acfb00ffcb8bbce9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/6421724e551d4560acfb00ffcb8bbce9 2024-11-16T04:25:01,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/3ddc16798b7e4a0389b841ffa299220d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/3ddc16798b7e4a0389b841ffa299220d 2024-11-16T04:25:01,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/37c84e42440543919759f0bc7b1ad6a0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/37c84e42440543919759f0bc7b1ad6a0 2024-11-16T04:25:01,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/ad62c1e94cdb4b6f90cc8b76d06658b4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/ad62c1e94cdb4b6f90cc8b76d06658b4 2024-11-16T04:25:01,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/9a989b6423da4c418e7d97f8cb315121 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/9a989b6423da4c418e7d97f8cb315121 2024-11-16T04:25:01,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/feadb24c1ef1470bb2a58bbc3d7d9766 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/feadb24c1ef1470bb2a58bbc3d7d9766 2024-11-16T04:25:01,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/24382fcb28e644869902364272973d55, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/395337ce8feb40e79a6f9ccdcbd03de7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/49a52e1447904fd8bd04816c43077f76, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/f762ad79134044fa8c205fee363feb8c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/18f7acd82f2040e7ad385ff01a8338f1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/53e3eb841b6b4c3a8bca18f0800a94a8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b8bf5737f7c04e649a8aee6a0b5d3bd7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7411ec2c3fdb4036ba8c599f719e0a8b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b223720d136747579c3a03edd77e6502, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d16ec634d9e44263a5994cc070f57a82, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/968dafcec3e74c588e3730fb315d6d00, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/a588a3eacbb449889b400410800b2c43, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/1bef56787e464a199b036b6d247ef568, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/fd04c994e62d4e51b453b8bc2b2ae187, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/010b0a9a21954dbfb1950f205d922562, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/cf7024e248104098a23150b795601703, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/867f3afb205d4cdabc328248966c03e5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d94f339d91b24b4f8fd568875a7c5701, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/4e789cfbcb07460790532bc9372228ba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8fd7ce4042c24e4c816802bd2fde74cd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/6fccb4683e96492a8b275ee7155523dd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/eca3db9224ba4bdf969828882066c165, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7bf782ce1e8846c6bd4aa2ee55834ebf, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36cdae2c85cf4b0aa9ae456a392c3085, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/582d4f9ecb974753a5191af870c1ac40, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/093704ba89a24b20a27964435271932e] to archive 2024-11-16T04:25:01,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:01,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36e0e55d2eca4dd9b15c2a5bd3ddc1f5 2024-11-16T04:25:01,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/24382fcb28e644869902364272973d55 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/24382fcb28e644869902364272973d55 2024-11-16T04:25:01,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/395337ce8feb40e79a6f9ccdcbd03de7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/395337ce8feb40e79a6f9ccdcbd03de7 2024-11-16T04:25:01,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/49a52e1447904fd8bd04816c43077f76 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/49a52e1447904fd8bd04816c43077f76 2024-11-16T04:25:01,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/f762ad79134044fa8c205fee363feb8c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/f762ad79134044fa8c205fee363feb8c 2024-11-16T04:25:01,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/18f7acd82f2040e7ad385ff01a8338f1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/18f7acd82f2040e7ad385ff01a8338f1 2024-11-16T04:25:01,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/53e3eb841b6b4c3a8bca18f0800a94a8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/53e3eb841b6b4c3a8bca18f0800a94a8 2024-11-16T04:25:01,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b8bf5737f7c04e649a8aee6a0b5d3bd7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b8bf5737f7c04e649a8aee6a0b5d3bd7 2024-11-16T04:25:01,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7411ec2c3fdb4036ba8c599f719e0a8b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7411ec2c3fdb4036ba8c599f719e0a8b 2024-11-16T04:25:01,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b223720d136747579c3a03edd77e6502 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/b223720d136747579c3a03edd77e6502 2024-11-16T04:25:01,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d16ec634d9e44263a5994cc070f57a82 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d16ec634d9e44263a5994cc070f57a82 2024-11-16T04:25:01,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/968dafcec3e74c588e3730fb315d6d00 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/968dafcec3e74c588e3730fb315d6d00 2024-11-16T04:25:01,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/a588a3eacbb449889b400410800b2c43 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/a588a3eacbb449889b400410800b2c43 2024-11-16T04:25:01,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/1bef56787e464a199b036b6d247ef568 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/1bef56787e464a199b036b6d247ef568 2024-11-16T04:25:01,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/fd04c994e62d4e51b453b8bc2b2ae187 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/fd04c994e62d4e51b453b8bc2b2ae187 2024-11-16T04:25:01,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/010b0a9a21954dbfb1950f205d922562 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/010b0a9a21954dbfb1950f205d922562 2024-11-16T04:25:01,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/cf7024e248104098a23150b795601703 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/cf7024e248104098a23150b795601703 2024-11-16T04:25:01,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/867f3afb205d4cdabc328248966c03e5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/867f3afb205d4cdabc328248966c03e5 2024-11-16T04:25:01,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d94f339d91b24b4f8fd568875a7c5701 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/d94f339d91b24b4f8fd568875a7c5701 2024-11-16T04:25:01,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/4e789cfbcb07460790532bc9372228ba to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/4e789cfbcb07460790532bc9372228ba 2024-11-16T04:25:01,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8fd7ce4042c24e4c816802bd2fde74cd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8fd7ce4042c24e4c816802bd2fde74cd 2024-11-16T04:25:01,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/6fccb4683e96492a8b275ee7155523dd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/6fccb4683e96492a8b275ee7155523dd 2024-11-16T04:25:01,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/eca3db9224ba4bdf969828882066c165 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/eca3db9224ba4bdf969828882066c165 2024-11-16T04:25:01,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7bf782ce1e8846c6bd4aa2ee55834ebf to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/7bf782ce1e8846c6bd4aa2ee55834ebf 2024-11-16T04:25:01,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36cdae2c85cf4b0aa9ae456a392c3085 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/36cdae2c85cf4b0aa9ae456a392c3085 2024-11-16T04:25:01,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/582d4f9ecb974753a5191af870c1ac40 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/582d4f9ecb974753a5191af870c1ac40 2024-11-16T04:25:01,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/093704ba89a24b20a27964435271932e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/093704ba89a24b20a27964435271932e 2024-11-16T04:25:01,215 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/recovered.edits/395.seqid, newMaxSeqId=395, maxSeqId=4 2024-11-16T04:25:01,216 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. 2024-11-16T04:25:01,216 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for add53441e0ab2ad8f2f1f109ed502c64: Waiting for close lock at 1731731099808Running coprocessor pre-close hooks at 1731731099808Disabling compacts and flushes for region at 1731731099808Disabling writes for close at 1731731099808Obtaining lock to block concurrent updates at 1731731099808Preparing flush snapshotting stores in add53441e0ab2ad8f2f1f109ed502c64 at 1731731099808Finished memstore snapshotting TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731731099809 (+1 ms)Flushing stores of TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64. at 1731731099809Flushing add53441e0ab2ad8f2f1f109ed502c64/A: creating writer at 1731731099809Flushing mob file add53441e0ab2ad8f2f1f109ed502c64/A: appending metadata at 1731731099816 (+7 ms)Flushing mob file add53441e0ab2ad8f2f1f109ed502c64/A: closing flushed file at 1731731099816Flushing add53441e0ab2ad8f2f1f109ed502c64/A: appending metadata at 1731731100235 (+419 ms)Flushing add53441e0ab2ad8f2f1f109ed502c64/A: closing flushed file at 1731731100235Flushing add53441e0ab2ad8f2f1f109ed502c64/B: creating writer at 1731731100656 (+421 ms)Flushing add53441e0ab2ad8f2f1f109ed502c64/B: appending metadata at 1731731100661 (+5 ms)Flushing add53441e0ab2ad8f2f1f109ed502c64/B: closing flushed file at 1731731100662 (+1 ms)Flushing add53441e0ab2ad8f2f1f109ed502c64/C: creating writer at 1731731101078 (+416 ms)Flushing add53441e0ab2ad8f2f1f109ed502c64/C: appending metadata at 1731731101081 (+3 ms)Flushing add53441e0ab2ad8f2f1f109ed502c64/C: closing flushed file at 1731731101081Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@146781f1: reopening flushed file at 1731731101089 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34e49b5c: reopening flushed file at 1731731101093 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7db47e3f: reopening flushed file at 1731731101098 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for add53441e0ab2ad8f2f1f109ed502c64 in 1295ms, sequenceid=392, compaction requested=false at 1731731101103 (+5 ms)Writing region close event to WAL at 1731731101212 (+109 ms)Running coprocessor post-close hooks at 1731731101215 (+3 ms)Closed at 1731731101216 (+1 ms) 2024-11-16T04:25:01,218 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,219 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=add53441e0ab2ad8f2f1f109ed502c64, regionState=CLOSED 2024-11-16T04:25:01,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:01,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T04:25:01,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure add53441e0ab2ad8f2f1f109ed502c64, server=fda3efb342d8,45455,1731731034087 in 1.5660 sec 2024-11-16T04:25:01,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T04:25:01,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=add53441e0ab2ad8f2f1f109ed502c64, UNASSIGN in 1.5710 sec 2024-11-16T04:25:01,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T04:25:01,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5730 sec 2024-11-16T04:25:01,227 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731101226"}]},"ts":"1731731101226"} 2024-11-16T04:25:01,228 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T04:25:01,228 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T04:25:01,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5830 sec 2024-11-16T04:25:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T04:25:01,794 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:01,795 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T04:25:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,797 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T04:25:01,798 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,801 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,806 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/recovered.edits] 2024-11-16T04:25:01,811 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4520f38968b649ccbf4156cbdad83368 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/4520f38968b649ccbf4156cbdad83368 2024-11-16T04:25:01,814 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c9dcb5b720ec473db088f5ea7c7d9763 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/A/c9dcb5b720ec473db088f5ea7c7d9763 2024-11-16T04:25:01,819 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/33d156bacb3e470f95b772aa5599b51b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/33d156bacb3e470f95b772aa5599b51b 2024-11-16T04:25:01,821 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/5f8ae7b0951148a4a572a7e338575d12 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/B/5f8ae7b0951148a4a572a7e338575d12 2024-11-16T04:25:01,825 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/60b4318282a4450fad0d8eb4f50be79d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/60b4318282a4450fad0d8eb4f50be79d 2024-11-16T04:25:01,827 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8546ab1abee54e13b1a4151e5bd550e6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/C/8546ab1abee54e13b1a4151e5bd550e6 2024-11-16T04:25:01,830 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/recovered.edits/395.seqid to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64/recovered.edits/395.seqid 2024-11-16T04:25:01,831 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,831 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T04:25:01,831 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T04:25:01,832 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T04:25:01,836 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160c5d76daf8084baa820a35ae81324879_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160c5d76daf8084baa820a35ae81324879_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,837 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111623d8be92692f4e4a87f42188a1fb1f19_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111623d8be92692f4e4a87f42188a1fb1f19_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,838 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164d62fa93a61643c2b63e8d051f08a147_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164d62fa93a61643c2b63e8d051f08a147_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,839 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111651cdb44880a547149d9700a27cc5e966_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111651cdb44880a547149d9700a27cc5e966_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,840 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116579ad89432fe46a79eb2cf1c28a88e26_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116579ad89432fe46a79eb2cf1c28a88e26_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,841 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165897faaa8e0d42d3b24a5d8768260315_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165897faaa8e0d42d3b24a5d8768260315_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,842 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165beb914436754ea8a78a6dccf07be552_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165beb914436754ea8a78a6dccf07be552_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,843 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166b5d871dc1f245b786cc17841da04be5_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166b5d871dc1f245b786cc17841da04be5_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,844 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111678af183b092f4b0582f6483819317089_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111678af183b092f4b0582f6483819317089_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,845 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116853cc34687344e12a2d3074b6594ba78_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116853cc34687344e12a2d3074b6594ba78_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,846 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168804e64462354c2d8cd61413a7fef879_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168804e64462354c2d8cd61413a7fef879_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,847 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a01d250da2c454bb5ea2f341fedeeb8_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168a01d250da2c454bb5ea2f341fedeeb8_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,848 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168f20a33ee8bb4c588ff0a1681feff37c_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168f20a33ee8bb4c588ff0a1681feff37c_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,848 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116af5919bb71a24887baae168a1dea288a_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116af5919bb71a24887baae168a1dea288a_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,849 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1f3d360e9bf4a27b68a334bc454ffd1_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b1f3d360e9bf4a27b68a334bc454ffd1_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,850 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b570bbc04a86479c88c6de699d83b7ac_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b570bbc04a86479c88c6de699d83b7ac_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,851 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d21e622c2fa34b8fba57dba99702538e_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d21e622c2fa34b8fba57dba99702538e_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,852 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e67c55177adc412a8e893bb050c9c615_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e67c55177adc412a8e893bb050c9c615_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,853 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea26ac1f071f4e19b27dc37764029b7b_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ea26ac1f071f4e19b27dc37764029b7b_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,853 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efd3fdf0f3dc46aeb5d574ffb5a20b4b_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efd3fdf0f3dc46aeb5d574ffb5a20b4b_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,854 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f90e3e3fdf574858b7e647de892a49f5_add53441e0ab2ad8f2f1f109ed502c64 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f90e3e3fdf574858b7e647de892a49f5_add53441e0ab2ad8f2f1f109ed502c64 2024-11-16T04:25:01,855 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T04:25:01,856 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,858 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T04:25:01,860 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T04:25:01,862 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,862 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T04:25:01,862 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731731101862"}]},"ts":"9223372036854775807"} 2024-11-16T04:25:01,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T04:25:01,864 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => add53441e0ab2ad8f2f1f109ed502c64, NAME => 'TestAcidGuarantees,,1731731065513.add53441e0ab2ad8f2f1f109ed502c64.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T04:25:01,865 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T04:25:01,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731731101865"}]},"ts":"9223372036854775807"} 2024-11-16T04:25:01,867 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T04:25:01,868 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-16T04:25:01,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-16T04:25:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T04:25:01,908 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:01,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=409 (was 400) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8506 (was 8466) - AvailableMemoryMB LEAK? - 2024-11-16T04:25:01,925 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=409, ProcessCount=11, AvailableMemoryMB=8504 2024-11-16T04:25:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:25:01,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:25:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:01,929 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T04:25:01,929 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:01,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-16T04:25:01,930 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T04:25:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T04:25:01,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742070_1246 (size=959) 2024-11-16T04:25:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T04:25:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T04:25:02,338 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:25:02,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742071_1247 (size=53) 2024-11-16T04:25:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T04:25:02,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:02,749 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6a049fe61dfb22dfb7ca7a9173671d57, disabling compactions & flushes 2024-11-16T04:25:02,749 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:02,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:02,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. after waiting 0 ms 2024-11-16T04:25:02,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:02,750 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:02,750 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6a049fe61dfb22dfb7ca7a9173671d57: Waiting for close lock at 1731731102749Disabling compacts and flushes for region at 1731731102749Disabling writes for close at 1731731102750 (+1 ms)Writing region close event to WAL at 1731731102750Closed at 1731731102750 2024-11-16T04:25:02,751 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T04:25:02,751 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731731102751"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731731102751"}]},"ts":"1731731102751"} 2024-11-16T04:25:02,753 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T04:25:02,754 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T04:25:02,754 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731102754"}]},"ts":"1731731102754"} 2024-11-16T04:25:02,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T04:25:02,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, ASSIGN}] 2024-11-16T04:25:02,757 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, ASSIGN 2024-11-16T04:25:02,758 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, ASSIGN; state=OFFLINE, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=false 2024-11-16T04:25:02,909 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6a049fe61dfb22dfb7ca7a9173671d57, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:02,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, ASSIGN because future has completed 2024-11-16T04:25:02,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T04:25:03,070 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:03,071 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:25:03,071 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,071 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:03,071 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,071 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,073 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,075 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a049fe61dfb22dfb7ca7a9173671d57 columnFamilyName A 2024-11-16T04:25:03,075 DEBUG [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:03,076 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] regionserver.HStore(327): Store=6a049fe61dfb22dfb7ca7a9173671d57/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:03,076 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,077 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a049fe61dfb22dfb7ca7a9173671d57 columnFamilyName B 2024-11-16T04:25:03,077 DEBUG [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:03,078 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] regionserver.HStore(327): Store=6a049fe61dfb22dfb7ca7a9173671d57/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:03,078 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,080 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a049fe61dfb22dfb7ca7a9173671d57 columnFamilyName C 2024-11-16T04:25:03,080 DEBUG [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:03,080 INFO [StoreOpener-6a049fe61dfb22dfb7ca7a9173671d57-1 {}] regionserver.HStore(327): Store=6a049fe61dfb22dfb7ca7a9173671d57/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:03,080 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,081 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,082 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,083 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,083 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,084 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:25:03,085 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,087 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:25:03,087 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 6a049fe61dfb22dfb7ca7a9173671d57; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74142890, jitterRate=0.10481515526771545}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:25:03,087 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:03,088 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 6a049fe61dfb22dfb7ca7a9173671d57: Running coprocessor pre-open hook at 1731731103071Writing region info on filesystem at 1731731103071Initializing all the Stores at 1731731103072 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731103072Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731103073 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731103073Cleaning up temporary data from old regions at 1731731103083 (+10 ms)Running coprocessor post-open hooks at 1731731103087 (+4 ms)Region opened successfully at 1731731103087 2024-11-16T04:25:03,088 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., pid=63, masterSystemTime=1731731103067 2024-11-16T04:25:03,090 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:03,090 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:03,091 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6a049fe61dfb22dfb7ca7a9173671d57, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:03,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:03,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T04:25:03,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 in 179 msec 2024-11-16T04:25:03,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-16T04:25:03,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, ASSIGN in 338 msec 2024-11-16T04:25:03,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T04:25:03,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731103097"}]},"ts":"1731731103097"} 2024-11-16T04:25:03,098 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T04:25:03,099 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T04:25:03,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1720 sec 2024-11-16T04:25:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-16T04:25:04,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:04,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f6c2d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,078 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,081 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd58b3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,085 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4905445e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,087 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,090 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:25:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:25:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@37284d9b 2024-11-16T04:25:04,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:25:04,092 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:25:04,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77284e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,094 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,094 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,094 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253c9db4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,096 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2b9a2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6071fcfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17fc8cfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,099 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bae9b08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259bf7d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,100 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,100 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,101 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45662154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,102 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668fbf7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdb05b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,103 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,103 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,104 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,104 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c48baf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,106 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70c970b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4a2b35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,108 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,108 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,109 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb770c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,110 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553bd2ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,111 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be8a148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,112 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,112 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b3a0f35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33a446b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7445538a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,115 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c3ef0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,116 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f38d64a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0b9ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,118 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85e6263, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,119 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2041cf60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,120 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605b153a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,120 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,120 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,120 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef03db0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,121 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,122 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@250208c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f53f1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:04,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:04,123 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c914835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:04,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:04,124 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b472de3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:04,125 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,125 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,125 DEBUG [Thread-1166 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,125 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,126 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,126 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,126 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,126 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,126 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,127 DEBUG [Thread-1164 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-16T04:25:04,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,128 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,129 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:04,130 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:04,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T04:25:04,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:04,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,132 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:04,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:04,142 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:04,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:25:04,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7a0fd48c9e25409d94582ac55f72ca1c is 50, key is test_row_0/A:col10/1731731104141/Put/seqid=0 2024-11-16T04:25:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731164170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731164174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731164176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731164176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731164177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742072_1248 (size=14341) 2024-11-16T04:25:04,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7a0fd48c9e25409d94582ac55f72ca1c 2024-11-16T04:25:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T04:25:04,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6be1773ea52143fba5fc86771c6e9345 is 50, key is test_row_0/B:col10/1731731104141/Put/seqid=0 2024-11-16T04:25:04,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742073_1249 (size=12001) 2024-11-16T04:25:04,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731164283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731164283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731164293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731164293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731164293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T04:25:04,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731164493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731164493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731164503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731164504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731164504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:04,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6be1773ea52143fba5fc86771c6e9345 2024-11-16T04:25:04,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/745ec44c4a3b4f4aba2b556a74bcafe0 is 50, key is test_row_0/C:col10/1731731104141/Put/seqid=0 2024-11-16T04:25:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742074_1250 (size=12001) 2024-11-16T04:25:04,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T04:25:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731164803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731164803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731164813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731164813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731164814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:04,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:04,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:05,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:05,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:05,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:05,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:05,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:05,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/745ec44c4a3b4f4aba2b556a74bcafe0 2024-11-16T04:25:05,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7a0fd48c9e25409d94582ac55f72ca1c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7a0fd48c9e25409d94582ac55f72ca1c 2024-11-16T04:25:05,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7a0fd48c9e25409d94582ac55f72ca1c, entries=200, sequenceid=13, filesize=14.0 K 2024-11-16T04:25:05,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6be1773ea52143fba5fc86771c6e9345 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6be1773ea52143fba5fc86771c6e9345 2024-11-16T04:25:05,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6be1773ea52143fba5fc86771c6e9345, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T04:25:05,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/745ec44c4a3b4f4aba2b556a74bcafe0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/745ec44c4a3b4f4aba2b556a74bcafe0 2024-11-16T04:25:05,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/745ec44c4a3b4f4aba2b556a74bcafe0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T04:25:05,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6a049fe61dfb22dfb7ca7a9173671d57 in 976ms, sequenceid=13, compaction requested=false 2024-11-16T04:25:05,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:05,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-16T04:25:05,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:05,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:25:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/524f2bf5f9714761b1c79ffb0fe18569 is 50, key is test_row_0/A:col10/1731731104174/Put/seqid=0 2024-11-16T04:25:05,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742075_1251 (size=12001) 2024-11-16T04:25:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T04:25:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:05,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:05,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731165324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731165324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731165325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731165326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731165327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731165433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731165433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731165433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,614 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/524f2bf5f9714761b1c79ffb0fe18569 2024-11-16T04:25:05,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/65b4b2de9fa942c2875239459ab53b35 is 50, key is test_row_0/B:col10/1731731104174/Put/seqid=0 2024-11-16T04:25:05,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742076_1252 (size=12001) 2024-11-16T04:25:05,631 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/65b4b2de9fa942c2875239459ab53b35 2024-11-16T04:25:05,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/6ea311a5eabe41d586ee24be561aed8b is 50, key is test_row_0/C:col10/1731731104174/Put/seqid=0 2024-11-16T04:25:05,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731165644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731165644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731165644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742077_1253 (size=12001) 2024-11-16T04:25:05,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731165953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731165953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731165953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:05,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,054 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/6ea311a5eabe41d586ee24be561aed8b 2024-11-16T04:25:06,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/524f2bf5f9714761b1c79ffb0fe18569 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/524f2bf5f9714761b1c79ffb0fe18569 2024-11-16T04:25:06,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/524f2bf5f9714761b1c79ffb0fe18569, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T04:25:06,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/65b4b2de9fa942c2875239459ab53b35 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/65b4b2de9fa942c2875239459ab53b35 2024-11-16T04:25:06,066 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/65b4b2de9fa942c2875239459ab53b35, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T04:25:06,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/6ea311a5eabe41d586ee24be561aed8b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6ea311a5eabe41d586ee24be561aed8b 2024-11-16T04:25:06,071 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6ea311a5eabe41d586ee24be561aed8b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T04:25:06,072 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6a049fe61dfb22dfb7ca7a9173671d57 in 868ms, sequenceid=37, compaction requested=false 2024-11-16T04:25:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-16T04:25:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-16T04:25:06,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T04:25:06,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9420 sec 2024-11-16T04:25:06,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9500 sec 2024-11-16T04:25:06,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-16T04:25:06,274 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:06,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T04:25:06,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:06,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:06,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:06,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:06,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:25:06,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7c40ffdc63bd4ff799ed0c6ca8db1da2 is 50, key is test_row_0/A:col10/1731731105325/Put/seqid=0 2024-11-16T04:25:06,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742078_1254 (size=14341) 2024-11-16T04:25:06,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:06,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731166384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731166385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731166464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731166464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731166464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731166493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731166493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:06,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731166703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731166703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:06,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:06,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:06,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7c40ffdc63bd4ff799ed0c6ca8db1da2 2024-11-16T04:25:06,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/aede92296500453fbb771ab200f5831a is 50, key is test_row_0/B:col10/1731731105325/Put/seqid=0 2024-11-16T04:25:06,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742079_1255 (size=12001) 2024-11-16T04:25:06,761 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:25:06,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:06,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:06,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:07,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731167013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731167013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:07,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/aede92296500453fbb771ab200f5831a 2024-11-16T04:25:07,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/e20d87cb94494aec961fa79f1568c32a is 50, key is test_row_0/C:col10/1731731105325/Put/seqid=0 2024-11-16T04:25:07,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742080_1256 (size=12001) 2024-11-16T04:25:07,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:07,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:07,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731167473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,476 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731167483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731167483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,486 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:07,486 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731167524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731167524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:07,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:07,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:07,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/e20d87cb94494aec961fa79f1568c32a 2024-11-16T04:25:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7c40ffdc63bd4ff799ed0c6ca8db1da2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7c40ffdc63bd4ff799ed0c6ca8db1da2 2024-11-16T04:25:07,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7c40ffdc63bd4ff799ed0c6ca8db1da2, entries=200, sequenceid=50, filesize=14.0 K 2024-11-16T04:25:07,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/aede92296500453fbb771ab200f5831a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/aede92296500453fbb771ab200f5831a 2024-11-16T04:25:07,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/aede92296500453fbb771ab200f5831a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T04:25:07,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/e20d87cb94494aec961fa79f1568c32a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/e20d87cb94494aec961fa79f1568c32a 2024-11-16T04:25:07,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/e20d87cb94494aec961fa79f1568c32a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-16T04:25:07,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1292ms, sequenceid=50, compaction requested=true 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:07,627 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:07,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T04:25:07,627 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:07,628 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:07,628 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:07,628 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:07,628 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:07,628 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,628 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,628 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7a0fd48c9e25409d94582ac55f72ca1c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/524f2bf5f9714761b1c79ffb0fe18569, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7c40ffdc63bd4ff799ed0c6ca8db1da2] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=39.7 K 2024-11-16T04:25:07,628 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6be1773ea52143fba5fc86771c6e9345, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/65b4b2de9fa942c2875239459ab53b35, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/aede92296500453fbb771ab200f5831a] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.2 K 2024-11-16T04:25:07,629 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be1773ea52143fba5fc86771c6e9345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731104138 2024-11-16T04:25:07,629 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0fd48c9e25409d94582ac55f72ca1c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731104138 2024-11-16T04:25:07,629 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b4b2de9fa942c2875239459ab53b35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731731104174 2024-11-16T04:25:07,629 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 524f2bf5f9714761b1c79ffb0fe18569, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731731104174 2024-11-16T04:25:07,629 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting aede92296500453fbb771ab200f5831a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731731105325 2024-11-16T04:25:07,629 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c40ffdc63bd4ff799ed0c6ca8db1da2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731731105323 2024-11-16T04:25:07,636 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:07,637 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/fd07d775dd354604a74fc942e1c6e1bb is 50, key is test_row_0/A:col10/1731731105325/Put/seqid=0 2024-11-16T04:25:07,639 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:07,640 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/63c2b5dd961c43de8c16e746b499b98c is 50, key is test_row_0/B:col10/1731731105325/Put/seqid=0 2024-11-16T04:25:07,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T04:25:07,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:25:07,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742082_1258 (size=12104) 2024-11-16T04:25:07,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742081_1257 (size=12104) 2024-11-16T04:25:07,676 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/fd07d775dd354604a74fc942e1c6e1bb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/fd07d775dd354604a74fc942e1c6e1bb 2024-11-16T04:25:07,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/b403b72d83e54ce1a8cdc1f2a4764c3d is 50, key is test_row_0/A:col10/1731731106384/Put/seqid=0 2024-11-16T04:25:07,684 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into fd07d775dd354604a74fc942e1c6e1bb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:07,684 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:07,684 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731107627; duration=0sec 2024-11-16T04:25:07,684 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:07,684 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:07,684 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:07,687 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:07,687 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:07,687 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:07,687 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/745ec44c4a3b4f4aba2b556a74bcafe0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6ea311a5eabe41d586ee24be561aed8b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/e20d87cb94494aec961fa79f1568c32a] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.2 K 2024-11-16T04:25:07,687 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 745ec44c4a3b4f4aba2b556a74bcafe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731104138 2024-11-16T04:25:07,688 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea311a5eabe41d586ee24be561aed8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731731104174 2024-11-16T04:25:07,688 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e20d87cb94494aec961fa79f1568c32a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731731105325 2024-11-16T04:25:07,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742083_1259 (size=12001) 2024-11-16T04:25:07,698 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:07,699 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/cdbf06a75ea4469abb147dde3dff3b0b is 50, key is test_row_0/C:col10/1731731105325/Put/seqid=0 2024-11-16T04:25:07,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742084_1260 (size=12104) 2024-11-16T04:25:07,718 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/cdbf06a75ea4469abb147dde3dff3b0b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/cdbf06a75ea4469abb147dde3dff3b0b 2024-11-16T04:25:07,722 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into cdbf06a75ea4469abb147dde3dff3b0b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:07,722 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:07,723 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731107627; duration=0sec 2024-11-16T04:25:07,723 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:07,723 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:08,067 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/63c2b5dd961c43de8c16e746b499b98c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/63c2b5dd961c43de8c16e746b499b98c 2024-11-16T04:25:08,072 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into 63c2b5dd961c43de8c16e746b499b98c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:08,072 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:08,072 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731107627; duration=0sec 2024-11-16T04:25:08,072 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:08,072 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:08,093 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/b403b72d83e54ce1a8cdc1f2a4764c3d 2024-11-16T04:25:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/4ee440c0e9cd4dd0879902616511c947 is 50, key is test_row_0/B:col10/1731731106384/Put/seqid=0 2024-11-16T04:25:08,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742085_1261 (size=12001) 2024-11-16T04:25:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:08,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/4ee440c0e9cd4dd0879902616511c947 2024-11-16T04:25:08,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ed5f8e46bbed41e793b742fd09b08530 is 50, key is test_row_0/C:col10/1731731106384/Put/seqid=0 2024-11-16T04:25:08,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742086_1262 (size=12001) 2024-11-16T04:25:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:08,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731168551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:08,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:08,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731168552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:08,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731168664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:08,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731168664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731168873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:08,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731168873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:08,923 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ed5f8e46bbed41e793b742fd09b08530 2024-11-16T04:25:08,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/b403b72d83e54ce1a8cdc1f2a4764c3d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b403b72d83e54ce1a8cdc1f2a4764c3d 2024-11-16T04:25:08,931 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b403b72d83e54ce1a8cdc1f2a4764c3d, entries=150, sequenceid=73, filesize=11.7 K 2024-11-16T04:25:08,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/4ee440c0e9cd4dd0879902616511c947 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4ee440c0e9cd4dd0879902616511c947 2024-11-16T04:25:08,936 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4ee440c0e9cd4dd0879902616511c947, entries=150, sequenceid=73, filesize=11.7 K 2024-11-16T04:25:08,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ed5f8e46bbed41e793b742fd09b08530 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ed5f8e46bbed41e793b742fd09b08530 2024-11-16T04:25:08,940 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ed5f8e46bbed41e793b742fd09b08530, entries=150, sequenceid=73, filesize=11.7 K 2024-11-16T04:25:08,941 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1289ms, sequenceid=73, compaction requested=false 2024-11-16T04:25:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:08,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T04:25:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T04:25:08,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T04:25:08,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6660 sec 2024-11-16T04:25:08,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6710 sec 2024-11-16T04:25:09,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:09,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T04:25:09,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3c32d082997f42c096ce025d19f22290 is 50, key is test_row_0/A:col10/1731731109186/Put/seqid=0 2024-11-16T04:25:09,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742087_1263 (size=14341) 2024-11-16T04:25:09,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731169222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731169224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731169333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731169333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731169483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,486 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731169494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,496 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731169513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,516 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731169543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731169543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3c32d082997f42c096ce025d19f22290 2024-11-16T04:25:09,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/9ad868c69d384b9f9419a01e8ae3db71 is 50, key is test_row_0/B:col10/1731731109186/Put/seqid=0 2024-11-16T04:25:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742088_1264 (size=12001) 2024-11-16T04:25:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731169853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:09,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731169853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:09,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:09,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:10,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/9ad868c69d384b9f9419a01e8ae3db71 2024-11-16T04:25:10,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/c9711d7c65a14cca89ae3819d26b23fa is 50, key is test_row_0/C:col10/1731731109186/Put/seqid=0 2024-11-16T04:25:10,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742089_1265 (size=12001) 2024-11-16T04:25:10,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:10,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731170363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:10,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:10,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/c9711d7c65a14cca89ae3819d26b23fa 2024-11-16T04:25:10,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3c32d082997f42c096ce025d19f22290 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3c32d082997f42c096ce025d19f22290 2024-11-16T04:25:10,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3c32d082997f42c096ce025d19f22290, entries=200, sequenceid=91, filesize=14.0 K 2024-11-16T04:25:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/9ad868c69d384b9f9419a01e8ae3db71 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/9ad868c69d384b9f9419a01e8ae3db71 2024-11-16T04:25:10,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/9ad868c69d384b9f9419a01e8ae3db71, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T04:25:10,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/c9711d7c65a14cca89ae3819d26b23fa as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c9711d7c65a14cca89ae3819d26b23fa 2024-11-16T04:25:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T04:25:10,443 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:10,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T04:25:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T04:25:10,445 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:10,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c9711d7c65a14cca89ae3819d26b23fa, entries=150, sequenceid=91, filesize=11.7 K 2024-11-16T04:25:10,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1259ms, sequenceid=91, compaction requested=true 2024-11-16T04:25:10,446 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:10,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:10,446 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:10,446 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:10,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:10,447 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:10,447 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:10,447 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:10,447 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:10,447 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:10,447 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/fd07d775dd354604a74fc942e1c6e1bb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b403b72d83e54ce1a8cdc1f2a4764c3d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3c32d082997f42c096ce025d19f22290] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=37.5 K 2024-11-16T04:25:10,447 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:10,448 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/63c2b5dd961c43de8c16e746b499b98c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4ee440c0e9cd4dd0879902616511c947, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/9ad868c69d384b9f9419a01e8ae3db71] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.3 K 2024-11-16T04:25:10,448 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 63c2b5dd961c43de8c16e746b499b98c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731731105325 2024-11-16T04:25:10,452 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ee440c0e9cd4dd0879902616511c947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731731106370 2024-11-16T04:25:10,452 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd07d775dd354604a74fc942e1c6e1bb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731731105325 2024-11-16T04:25:10,453 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b403b72d83e54ce1a8cdc1f2a4764c3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731731106370 2024-11-16T04:25:10,453 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad868c69d384b9f9419a01e8ae3db71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731731108542 2024-11-16T04:25:10,453 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c32d082997f42c096ce025d19f22290, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731731108542 2024-11-16T04:25:10,460 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:10,460 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/0726916680a64fcd9adf05612cb87e90 is 50, key is test_row_0/B:col10/1731731109186/Put/seqid=0 2024-11-16T04:25:10,461 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:10,462 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/1f594bbcb56c45599387e99e4b0768b4 is 50, key is test_row_0/A:col10/1731731109186/Put/seqid=0 2024-11-16T04:25:10,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742091_1267 (size=12207) 2024-11-16T04:25:10,482 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/1f594bbcb56c45599387e99e4b0768b4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1f594bbcb56c45599387e99e4b0768b4 2024-11-16T04:25:10,487 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into 1f594bbcb56c45599387e99e4b0768b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:10,488 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:10,488 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731110446; duration=0sec 2024-11-16T04:25:10,488 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:10,488 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:10,488 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:10,489 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:10,489 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:10,489 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:10,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742090_1266 (size=12207) 2024-11-16T04:25:10,489 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/cdbf06a75ea4469abb147dde3dff3b0b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ed5f8e46bbed41e793b742fd09b08530, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c9711d7c65a14cca89ae3819d26b23fa] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.3 K 2024-11-16T04:25:10,490 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdbf06a75ea4469abb147dde3dff3b0b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731731105325 2024-11-16T04:25:10,490 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5f8e46bbed41e793b742fd09b08530, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731731106370 2024-11-16T04:25:10,490 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9711d7c65a14cca89ae3819d26b23fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731731108542 2024-11-16T04:25:10,495 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/0726916680a64fcd9adf05612cb87e90 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/0726916680a64fcd9adf05612cb87e90 2024-11-16T04:25:10,497 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:10,497 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7fb40d106f214974bcf606640e84b7e1 is 50, key is test_row_0/C:col10/1731731109186/Put/seqid=0 2024-11-16T04:25:10,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742092_1268 (size=12207) 2024-11-16T04:25:10,502 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into 0726916680a64fcd9adf05612cb87e90(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:10,502 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:10,502 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731110446; duration=0sec 2024-11-16T04:25:10,502 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:10,502 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:10,506 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7fb40d106f214974bcf606640e84b7e1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7fb40d106f214974bcf606640e84b7e1 2024-11-16T04:25:10,512 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into 7fb40d106f214974bcf606640e84b7e1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:10,512 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:10,512 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731110446; duration=0sec 2024-11-16T04:25:10,512 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:10,512 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T04:25:10,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T04:25:10,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:10,598 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:25:10,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/04b392fc224a4a79814a709d7dca1636 is 50, key is test_row_0/A:col10/1731731109221/Put/seqid=0 2024-11-16T04:25:10,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742093_1269 (size=12001) 2024-11-16T04:25:10,607 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/04b392fc224a4a79814a709d7dca1636 2024-11-16T04:25:10,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/4a242c83cdf6415a9d7d0c3e8620ae7e is 50, key is test_row_0/B:col10/1731731109221/Put/seqid=0 2024-11-16T04:25:10,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742094_1270 (size=12001) 2024-11-16T04:25:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T04:25:11,051 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/4a242c83cdf6415a9d7d0c3e8620ae7e 2024-11-16T04:25:11,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/363fdd88923240bab68b902bf99b7eba is 50, key is test_row_0/C:col10/1731731109221/Put/seqid=0 2024-11-16T04:25:11,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742095_1271 (size=12001) 2024-11-16T04:25:11,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T04:25:11,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:11,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731171395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731171399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/363fdd88923240bab68b902bf99b7eba 2024-11-16T04:25:11,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/04b392fc224a4a79814a709d7dca1636 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/04b392fc224a4a79814a709d7dca1636 2024-11-16T04:25:11,469 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/04b392fc224a4a79814a709d7dca1636, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:25:11,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/4a242c83cdf6415a9d7d0c3e8620ae7e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4a242c83cdf6415a9d7d0c3e8620ae7e 2024-11-16T04:25:11,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4a242c83cdf6415a9d7d0c3e8620ae7e, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:25:11,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/363fdd88923240bab68b902bf99b7eba as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/363fdd88923240bab68b902bf99b7eba 2024-11-16T04:25:11,478 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/363fdd88923240bab68b902bf99b7eba, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:25:11,479 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6a049fe61dfb22dfb7ca7a9173671d57 in 881ms, sequenceid=115, compaction requested=false 2024-11-16T04:25:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T04:25:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T04:25:11,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T04:25:11,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-11-16T04:25:11,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0390 sec 2024-11-16T04:25:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:11,506 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:11,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T04:25:11,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/d7e0733a5d974471a6724309f44bdf94 is 50, key is test_row_0/A:col10/1731731111397/Put/seqid=0 2024-11-16T04:25:11,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742096_1272 (size=12101) 2024-11-16T04:25:11,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731171551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731171551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T04:25:11,584 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:11,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T04:25:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T04:25:11,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:11,593 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:11,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:11,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731171663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731171663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T04:25:11,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:11,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731171873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:11,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731171873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:11,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:11,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:11,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T04:25:11,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/d7e0733a5d974471a6724309f44bdf94 2024-11-16T04:25:11,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/efd406a4048e4309845a5113499946e5 is 50, key is test_row_0/B:col10/1731731111397/Put/seqid=0 2024-11-16T04:25:11,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742097_1273 (size=12101) 2024-11-16T04:25:12,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731172183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:12,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731172183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:12,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T04:25:12,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/efd406a4048e4309845a5113499946e5 2024-11-16T04:25:12,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/b934f055fb9d47b994d58285687818ef is 50, key is test_row_0/C:col10/1731731111397/Put/seqid=0 2024-11-16T04:25:12,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742098_1274 (size=12101) 2024-11-16T04:25:12,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:12,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731172693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:12,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731172693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:12,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T04:25:12,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/b934f055fb9d47b994d58285687818ef 2024-11-16T04:25:12,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/d7e0733a5d974471a6724309f44bdf94 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7e0733a5d974471a6724309f44bdf94 2024-11-16T04:25:12,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7e0733a5d974471a6724309f44bdf94, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T04:25:12,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/efd406a4048e4309845a5113499946e5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/efd406a4048e4309845a5113499946e5 2024-11-16T04:25:12,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/efd406a4048e4309845a5113499946e5, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T04:25:12,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/b934f055fb9d47b994d58285687818ef as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/b934f055fb9d47b994d58285687818ef 2024-11-16T04:25:12,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/b934f055fb9d47b994d58285687818ef, entries=150, sequenceid=131, filesize=11.8 K 2024-11-16T04:25:12,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1254ms, sequenceid=131, compaction requested=true 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:12,760 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:12,760 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:12,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:12,761 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:12,761 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:12,761 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:12,761 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,761 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:12,761 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,761 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1f594bbcb56c45599387e99e4b0768b4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/04b392fc224a4a79814a709d7dca1636, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7e0733a5d974471a6724309f44bdf94] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.5 K 2024-11-16T04:25:12,761 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/0726916680a64fcd9adf05612cb87e90, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4a242c83cdf6415a9d7d0c3e8620ae7e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/efd406a4048e4309845a5113499946e5] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.5 K 2024-11-16T04:25:12,762 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f594bbcb56c45599387e99e4b0768b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731731108542 2024-11-16T04:25:12,762 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0726916680a64fcd9adf05612cb87e90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731731108542 2024-11-16T04:25:12,762 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04b392fc224a4a79814a709d7dca1636, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731109210 2024-11-16T04:25:12,762 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a242c83cdf6415a9d7d0c3e8620ae7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731109210 2024-11-16T04:25:12,763 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting efd406a4048e4309845a5113499946e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731731111390 2024-11-16T04:25:12,763 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7e0733a5d974471a6724309f44bdf94, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731731111390 2024-11-16T04:25:12,769 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:12,769 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:12,770 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/e7d107e8609e42e0b4e8dbea2d8e9a2b is 50, key is test_row_0/A:col10/1731731111397/Put/seqid=0 2024-11-16T04:25:12,770 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/17c12ec1a6c44d4a996f4a1981e0ecda is 50, key is test_row_0/B:col10/1731731111397/Put/seqid=0 2024-11-16T04:25:12,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742099_1275 (size=12409) 2024-11-16T04:25:12,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742100_1276 (size=12409) 2024-11-16T04:25:12,794 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/e7d107e8609e42e0b4e8dbea2d8e9a2b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/e7d107e8609e42e0b4e8dbea2d8e9a2b 2024-11-16T04:25:12,798 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into e7d107e8609e42e0b4e8dbea2d8e9a2b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:12,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:12,799 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731112760; duration=0sec 2024-11-16T04:25:12,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:12,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:12,799 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:12,800 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:12,800 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:12,800 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,800 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7fb40d106f214974bcf606640e84b7e1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/363fdd88923240bab68b902bf99b7eba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/b934f055fb9d47b994d58285687818ef] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.5 K 2024-11-16T04:25:12,800 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fb40d106f214974bcf606640e84b7e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731731108542 2024-11-16T04:25:12,800 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 363fdd88923240bab68b902bf99b7eba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731109210 2024-11-16T04:25:12,801 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b934f055fb9d47b994d58285687818ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731731111390 2024-11-16T04:25:12,806 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:12,806 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/08a558bea2f948939003192719369b6a is 50, key is test_row_0/C:col10/1731731111397/Put/seqid=0 2024-11-16T04:25:12,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742101_1277 (size=12409) 2024-11-16T04:25:12,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T04:25:12,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:12,815 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:25:12,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/5548c3dab564469984dfc331cda54319 is 50, key is test_row_0/A:col10/1731731111549/Put/seqid=0 2024-11-16T04:25:12,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742102_1278 (size=12151) 2024-11-16T04:25:12,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/5548c3dab564469984dfc331cda54319 2024-11-16T04:25:12,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/28ba1a6ca54445c28eeb010f7962ec03 is 50, key is test_row_0/B:col10/1731731111549/Put/seqid=0 2024-11-16T04:25:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742103_1279 (size=12151) 2024-11-16T04:25:12,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/28ba1a6ca54445c28eeb010f7962ec03 2024-11-16T04:25:12,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ea76fc96f8b645128169ab5d4b6d5532 is 50, key is test_row_0/C:col10/1731731111549/Put/seqid=0 2024-11-16T04:25:12,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742104_1280 (size=12151) 2024-11-16T04:25:13,186 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/17c12ec1a6c44d4a996f4a1981e0ecda as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/17c12ec1a6c44d4a996f4a1981e0ecda 2024-11-16T04:25:13,191 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into 17c12ec1a6c44d4a996f4a1981e0ecda(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:13,191 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:13,191 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731112760; duration=0sec 2024-11-16T04:25:13,191 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:13,191 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:13,218 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/08a558bea2f948939003192719369b6a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/08a558bea2f948939003192719369b6a 2024-11-16T04:25:13,222 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into 08a558bea2f948939003192719369b6a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:13,222 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:13,223 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731112760; duration=0sec 2024-11-16T04:25:13,223 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:13,223 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:13,290 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ea76fc96f8b645128169ab5d4b6d5532 2024-11-16T04:25:13,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/5548c3dab564469984dfc331cda54319 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/5548c3dab564469984dfc331cda54319 2024-11-16T04:25:13,298 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/5548c3dab564469984dfc331cda54319, entries=150, sequenceid=153, filesize=11.9 K 2024-11-16T04:25:13,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/28ba1a6ca54445c28eeb010f7962ec03 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/28ba1a6ca54445c28eeb010f7962ec03 2024-11-16T04:25:13,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/28ba1a6ca54445c28eeb010f7962ec03, entries=150, sequenceid=153, filesize=11.9 K 2024-11-16T04:25:13,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ea76fc96f8b645128169ab5d4b6d5532 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea76fc96f8b645128169ab5d4b6d5532 2024-11-16T04:25:13,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea76fc96f8b645128169ab5d4b6d5532, entries=150, sequenceid=153, filesize=11.9 K 2024-11-16T04:25:13,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 6a049fe61dfb22dfb7ca7a9173671d57 in 493ms, sequenceid=153, compaction requested=false 2024-11-16T04:25:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:13,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T04:25:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T04:25:13,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T04:25:13,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-16T04:25:13,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7290 sec 2024-11-16T04:25:13,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:13,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:25:13,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/442fe06680a04cd4b3fafec3975651a3 is 50, key is test_row_0/A:col10/1731731113511/Put/seqid=0 2024-11-16T04:25:13,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742105_1281 (size=14541) 2024-11-16T04:25:13,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/442fe06680a04cd4b3fafec3975651a3 2024-11-16T04:25:13,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/275d3736d08a44f0aa5b4fdb383e0294 is 50, key is test_row_0/B:col10/1731731113511/Put/seqid=0 2024-11-16T04:25:13,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742106_1282 (size=12151) 2024-11-16T04:25:13,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731173563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731173564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731173565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731173673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731173673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731173674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731173703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,705 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731173713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,716 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T04:25:13,734 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:13,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T04:25:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T04:25:13,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:13,736 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:13,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:13,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T04:25:13,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731173884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:13,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:13,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T04:25:13,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:13,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:13,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:13,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:13,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:13,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/275d3736d08a44f0aa5b4fdb383e0294 2024-11-16T04:25:13,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/51ed6410a82e48c298d43aa5117e2c7a is 50, key is test_row_0/C:col10/1731731113511/Put/seqid=0 2024-11-16T04:25:13,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742107_1283 (size=12151) 2024-11-16T04:25:14,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T04:25:14,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T04:25:14,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T04:25:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731174193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731174194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731174194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T04:25:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T04:25:14,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/51ed6410a82e48c298d43aa5117e2c7a 2024-11-16T04:25:14,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/442fe06680a04cd4b3fafec3975651a3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/442fe06680a04cd4b3fafec3975651a3 2024-11-16T04:25:14,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/442fe06680a04cd4b3fafec3975651a3, entries=200, sequenceid=166, filesize=14.2 K 2024-11-16T04:25:14,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/275d3736d08a44f0aa5b4fdb383e0294 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/275d3736d08a44f0aa5b4fdb383e0294 2024-11-16T04:25:14,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/275d3736d08a44f0aa5b4fdb383e0294, entries=150, sequenceid=166, filesize=11.9 K 2024-11-16T04:25:14,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/51ed6410a82e48c298d43aa5117e2c7a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/51ed6410a82e48c298d43aa5117e2c7a 2024-11-16T04:25:14,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/51ed6410a82e48c298d43aa5117e2c7a, entries=150, sequenceid=166, filesize=11.9 K 2024-11-16T04:25:14,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6a049fe61dfb22dfb7ca7a9173671d57 in 868ms, sequenceid=166, compaction requested=true 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:14,384 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:14,384 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:14,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:14,386 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:14,386 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:14,386 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:14,386 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:14,386 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,387 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/e7d107e8609e42e0b4e8dbea2d8e9a2b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/5548c3dab564469984dfc331cda54319, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/442fe06680a04cd4b3fafec3975651a3] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=38.2 K 2024-11-16T04:25:14,387 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,387 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/17c12ec1a6c44d4a996f4a1981e0ecda, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/28ba1a6ca54445c28eeb010f7962ec03, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/275d3736d08a44f0aa5b4fdb383e0294] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.9 K 2024-11-16T04:25:14,387 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7d107e8609e42e0b4e8dbea2d8e9a2b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731731111390 2024-11-16T04:25:14,387 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c12ec1a6c44d4a996f4a1981e0ecda, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731731111390 2024-11-16T04:25:14,387 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5548c3dab564469984dfc331cda54319, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731731111545 2024-11-16T04:25:14,387 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 442fe06680a04cd4b3fafec3975651a3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731731113510 2024-11-16T04:25:14,387 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ba1a6ca54445c28eeb010f7962ec03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731731111545 2024-11-16T04:25:14,388 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 275d3736d08a44f0aa5b4fdb383e0294, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731731113511 2024-11-16T04:25:14,394 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:14,394 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:14,394 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/fa1107e4e6b84969ad4ca9e118a034bb is 50, key is test_row_0/B:col10/1731731113511/Put/seqid=0 2024-11-16T04:25:14,395 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/77c8d99710704ab09863e38f649907e8 is 50, key is test_row_0/A:col10/1731731113511/Put/seqid=0 2024-11-16T04:25:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742108_1284 (size=12561) 2024-11-16T04:25:14,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742109_1285 (size=12561) 2024-11-16T04:25:14,416 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/fa1107e4e6b84969ad4ca9e118a034bb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/fa1107e4e6b84969ad4ca9e118a034bb 2024-11-16T04:25:14,421 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into fa1107e4e6b84969ad4ca9e118a034bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:14,421 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:14,421 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731114384; duration=0sec 2024-11-16T04:25:14,421 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:14,421 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:14,422 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:14,422 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:14,422 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:14,423 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,423 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/08a558bea2f948939003192719369b6a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea76fc96f8b645128169ab5d4b6d5532, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/51ed6410a82e48c298d43aa5117e2c7a] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=35.9 K 2024-11-16T04:25:14,423 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a558bea2f948939003192719369b6a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731731111390 2024-11-16T04:25:14,423 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ea76fc96f8b645128169ab5d4b6d5532, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731731111545 2024-11-16T04:25:14,424 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ed6410a82e48c298d43aa5117e2c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731731113511 2024-11-16T04:25:14,429 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:14,429 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/f50a89c8080045e9926ab933acc1967e is 50, key is test_row_0/C:col10/1731731113511/Put/seqid=0 2024-11-16T04:25:14,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742110_1286 (size=12561) 2024-11-16T04:25:14,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T04:25:14,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:14,500 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:25:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e is 50, key is test_row_0/A:col10/1731731113563/Put/seqid=0 2024-11-16T04:25:14,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742111_1287 (size=12151) 2024-11-16T04:25:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731174714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731174715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731174717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,817 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/77c8d99710704ab09863e38f649907e8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/77c8d99710704ab09863e38f649907e8 2024-11-16T04:25:14,821 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into 77c8d99710704ab09863e38f649907e8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:14,821 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:14,821 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731114384; duration=0sec 2024-11-16T04:25:14,821 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:14,821 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731174823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731174824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:14,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731174824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:14,837 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/f50a89c8080045e9926ab933acc1967e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f50a89c8080045e9926ab933acc1967e 2024-11-16T04:25:14,840 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into f50a89c8080045e9926ab933acc1967e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:14,840 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:14,840 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731114384; duration=0sec 2024-11-16T04:25:14,840 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:14,840 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T04:25:14,908 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e 2024-11-16T04:25:14,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/bc57d90d4b164697924f2956f0e496f7 is 50, key is test_row_0/B:col10/1731731113563/Put/seqid=0 2024-11-16T04:25:14,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742112_1288 (size=12151) 2024-11-16T04:25:15,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731175033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731175033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731175033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,322 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/bc57d90d4b164697924f2956f0e496f7 2024-11-16T04:25:15,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/256a31ef51fd4ee5ac9c97cf3501fb66 is 50, key is test_row_0/C:col10/1731731113563/Put/seqid=0 2024-11-16T04:25:15,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742113_1289 (size=12151) 2024-11-16T04:25:15,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731175344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731175344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731175344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731175724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,725 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,733 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/256a31ef51fd4ee5ac9c97cf3501fb66 2024-11-16T04:25:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731175733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,735 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e 2024-11-16T04:25:15,743 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T04:25:15,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/bc57d90d4b164697924f2956f0e496f7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/bc57d90d4b164697924f2956f0e496f7 2024-11-16T04:25:15,747 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/bc57d90d4b164697924f2956f0e496f7, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T04:25:15,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/256a31ef51fd4ee5ac9c97cf3501fb66 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/256a31ef51fd4ee5ac9c97cf3501fb66 2024-11-16T04:25:15,752 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/256a31ef51fd4ee5ac9c97cf3501fb66, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T04:25:15,752 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1252ms, sequenceid=192, compaction requested=false 2024-11-16T04:25:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:15,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:15,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T04:25:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T04:25:15,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T04:25:15,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-11-16T04:25:15,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0220 sec 2024-11-16T04:25:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:15,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:25:15,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/29bface4c73c4a04892bced0e2e746fd is 50, key is test_row_0/A:col10/1731731115855/Put/seqid=0 2024-11-16T04:25:15,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742114_1290 (size=14541) 2024-11-16T04:25:15,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/29bface4c73c4a04892bced0e2e746fd 2024-11-16T04:25:15,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6ac801baa07f48bbae5fb3d6991e2641 is 50, key is test_row_0/B:col10/1731731115855/Put/seqid=0 2024-11-16T04:25:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T04:25:15,884 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:15,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:15,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T04:25:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:15,887 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:15,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742115_1291 (size=12151) 2024-11-16T04:25:15,893 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:15,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731175897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731175903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:15,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731175904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:15,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731176013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731176014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731176014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T04:25:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T04:25:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731176223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731176224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731176224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6ac801baa07f48bbae5fb3d6991e2641 2024-11-16T04:25:16,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/4dc5279198a042e49a34d205b1d84a71 is 50, key is test_row_0/C:col10/1731731115855/Put/seqid=0 2024-11-16T04:25:16,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742116_1292 (size=12151) 2024-11-16T04:25:16,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T04:25:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T04:25:16,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:16,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731176533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731176533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731176533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:16,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:16,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T04:25:16,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:16,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:16,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/4dc5279198a042e49a34d205b1d84a71 2024-11-16T04:25:16,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/29bface4c73c4a04892bced0e2e746fd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/29bface4c73c4a04892bced0e2e746fd 2024-11-16T04:25:16,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/29bface4c73c4a04892bced0e2e746fd, entries=200, sequenceid=208, filesize=14.2 K 2024-11-16T04:25:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6ac801baa07f48bbae5fb3d6991e2641 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6ac801baa07f48bbae5fb3d6991e2641 2024-11-16T04:25:16,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6ac801baa07f48bbae5fb3d6991e2641, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T04:25:16,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/4dc5279198a042e49a34d205b1d84a71 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/4dc5279198a042e49a34d205b1d84a71 2024-11-16T04:25:16,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/4dc5279198a042e49a34d205b1d84a71, entries=150, sequenceid=208, filesize=11.9 K 2024-11-16T04:25:16,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 6a049fe61dfb22dfb7ca7a9173671d57 in 866ms, sequenceid=208, compaction requested=true 2024-11-16T04:25:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:16,721 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:16,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:16,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:16,722 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:16,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:16,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:16,722 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:16,722 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:16,722 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:16,722 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:16,722 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,723 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/77c8d99710704ab09863e38f649907e8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/29bface4c73c4a04892bced0e2e746fd] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=38.3 K 2024-11-16T04:25:16,723 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,723 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/fa1107e4e6b84969ad4ca9e118a034bb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/bc57d90d4b164697924f2956f0e496f7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6ac801baa07f48bbae5fb3d6991e2641] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.0 K 2024-11-16T04:25:16,723 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77c8d99710704ab09863e38f649907e8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731731113511 2024-11-16T04:25:16,723 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1107e4e6b84969ad4ca9e118a034bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731731113511 2024-11-16T04:25:16,723 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7cd7bc064d34ba8bb93d9b91b0c2d9e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731731113562 2024-11-16T04:25:16,723 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting bc57d90d4b164697924f2956f0e496f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731731113562 2024-11-16T04:25:16,724 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29bface4c73c4a04892bced0e2e746fd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731731114710 2024-11-16T04:25:16,724 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac801baa07f48bbae5fb3d6991e2641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731731114710 2024-11-16T04:25:16,730 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:16,730 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:16,730 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/5c31f02845364a4bbae86d7b38604b67 is 50, key is test_row_0/B:col10/1731731115855/Put/seqid=0 2024-11-16T04:25:16,731 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/a30e3eff179d401bbad2b90c9d453192 is 50, key is test_row_0/A:col10/1731731115855/Put/seqid=0 2024-11-16T04:25:16,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742117_1293 (size=12663) 2024-11-16T04:25:16,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742118_1294 (size=12663) 2024-11-16T04:25:16,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T04:25:16,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:16,809 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:25:16,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/18ed3187a6a2428ba645d8f30f0cf8d3 is 50, key is test_row_0/A:col10/1731731115902/Put/seqid=0 2024-11-16T04:25:16,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742119_1295 (size=12151) 2024-11-16T04:25:17,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:17,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:17,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:17,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731177057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731177062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731177062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,146 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/5c31f02845364a4bbae86d7b38604b67 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/5c31f02845364a4bbae86d7b38604b67 2024-11-16T04:25:17,150 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into 5c31f02845364a4bbae86d7b38604b67(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:17,150 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:17,150 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731116721; duration=0sec 2024-11-16T04:25:17,150 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:17,150 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:17,151 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:17,151 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:17,151 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:17,151 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:17,152 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f50a89c8080045e9926ab933acc1967e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/256a31ef51fd4ee5ac9c97cf3501fb66, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/4dc5279198a042e49a34d205b1d84a71] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.0 K 2024-11-16T04:25:17,152 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f50a89c8080045e9926ab933acc1967e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731731113511 2024-11-16T04:25:17,152 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 256a31ef51fd4ee5ac9c97cf3501fb66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731731113562 2024-11-16T04:25:17,153 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dc5279198a042e49a34d205b1d84a71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731731114710 2024-11-16T04:25:17,155 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/a30e3eff179d401bbad2b90c9d453192 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/a30e3eff179d401bbad2b90c9d453192 2024-11-16T04:25:17,159 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:17,159 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into a30e3eff179d401bbad2b90c9d453192(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:17,159 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:17,160 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731116721; duration=0sec 2024-11-16T04:25:17,160 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:17,160 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:17,160 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7f0b2871606847a582cd057daef6a7fe is 50, key is test_row_0/C:col10/1731731115855/Put/seqid=0 2024-11-16T04:25:17,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742120_1296 (size=12663) 2024-11-16T04:25:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731177163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731177174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731177174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,216 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/18ed3187a6a2428ba645d8f30f0cf8d3 2024-11-16T04:25:17,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/7e14344ddb454e2791f1d745a6914ef5 is 50, key is test_row_0/B:col10/1731731115902/Put/seqid=0 2024-11-16T04:25:17,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742121_1297 (size=12151) 2024-11-16T04:25:17,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731177373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731177384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731177384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,567 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7f0b2871606847a582cd057daef6a7fe as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f0b2871606847a582cd057daef6a7fe 2024-11-16T04:25:17,571 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into 7f0b2871606847a582cd057daef6a7fe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:17,571 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:17,571 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731116722; duration=0sec 2024-11-16T04:25:17,571 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:17,571 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:17,628 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/7e14344ddb454e2791f1d745a6914ef5 2024-11-16T04:25:17,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/fed81d56472740a4b9002b37dba7f966 is 50, key is test_row_0/C:col10/1731731115902/Put/seqid=0 2024-11-16T04:25:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742122_1298 (size=12151) 2024-11-16T04:25:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731177683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731177693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:17,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731177693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:18,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/fed81d56472740a4b9002b37dba7f966 2024-11-16T04:25:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/18ed3187a6a2428ba645d8f30f0cf8d3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/18ed3187a6a2428ba645d8f30f0cf8d3 2024-11-16T04:25:18,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/18ed3187a6a2428ba645d8f30f0cf8d3, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T04:25:18,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/7e14344ddb454e2791f1d745a6914ef5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/7e14344ddb454e2791f1d745a6914ef5 2024-11-16T04:25:18,048 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/7e14344ddb454e2791f1d745a6914ef5, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T04:25:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/fed81d56472740a4b9002b37dba7f966 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/fed81d56472740a4b9002b37dba7f966 2024-11-16T04:25:18,052 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/fed81d56472740a4b9002b37dba7f966, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T04:25:18,053 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1244ms, sequenceid=231, compaction requested=false 2024-11-16T04:25:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:18,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T04:25:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T04:25:18,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T04:25:18,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-16T04:25:18,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1730 sec 2024-11-16T04:25:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:18,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:18,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:25:18,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3eb8e5137d384e41850ade9084bd7105 is 50, key is test_row_0/A:col10/1731731117061/Put/seqid=0 2024-11-16T04:25:18,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742123_1299 (size=14541) 2024-11-16T04:25:18,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731178222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731178223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731178227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731178333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731178333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731178333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731178543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731178543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731178543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3eb8e5137d384e41850ade9084bd7105 2024-11-16T04:25:18,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/284bd80fd1d3420aa28cecff3fce8298 is 50, key is test_row_0/B:col10/1731731117061/Put/seqid=0 2024-11-16T04:25:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742124_1300 (size=12151) 2024-11-16T04:25:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731178853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731178854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731178854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:18,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:18,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/284bd80fd1d3420aa28cecff3fce8298 2024-11-16T04:25:19,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7d6e368631294b23a602420d29c185d2 is 50, key is test_row_0/C:col10/1731731117061/Put/seqid=0 2024-11-16T04:25:19,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742125_1301 (size=12151) 2024-11-16T04:25:19,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731179364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731179364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731179364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7d6e368631294b23a602420d29c185d2 2024-11-16T04:25:19,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3eb8e5137d384e41850ade9084bd7105 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3eb8e5137d384e41850ade9084bd7105 2024-11-16T04:25:19,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3eb8e5137d384e41850ade9084bd7105, entries=200, sequenceid=249, filesize=14.2 K 2024-11-16T04:25:19,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/284bd80fd1d3420aa28cecff3fce8298 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/284bd80fd1d3420aa28cecff3fce8298 2024-11-16T04:25:19,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/284bd80fd1d3420aa28cecff3fce8298, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T04:25:19,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7d6e368631294b23a602420d29c185d2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7d6e368631294b23a602420d29c185d2 2024-11-16T04:25:19,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7d6e368631294b23a602420d29c185d2, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T04:25:19,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1284ms, sequenceid=249, compaction requested=true 2024-11-16T04:25:19,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:19,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:19,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:19,480 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:19,480 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:19,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:19,481 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:19,481 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:19,482 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:19,482 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/5c31f02845364a4bbae86d7b38604b67, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/7e14344ddb454e2791f1d745a6914ef5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/284bd80fd1d3420aa28cecff3fce8298] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.1 K 2024-11-16T04:25:19,482 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:19,482 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:19,482 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:19,482 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/a30e3eff179d401bbad2b90c9d453192, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/18ed3187a6a2428ba645d8f30f0cf8d3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3eb8e5137d384e41850ade9084bd7105] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=38.4 K 2024-11-16T04:25:19,483 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a30e3eff179d401bbad2b90c9d453192, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731731114710 2024-11-16T04:25:19,483 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c31f02845364a4bbae86d7b38604b67, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731731114710 2024-11-16T04:25:19,484 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e14344ddb454e2791f1d745a6914ef5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731731115882 2024-11-16T04:25:19,484 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ed3187a6a2428ba645d8f30f0cf8d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731731115882 2024-11-16T04:25:19,485 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb8e5137d384e41850ade9084bd7105, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731117056 2024-11-16T04:25:19,486 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 284bd80fd1d3420aa28cecff3fce8298, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731117056 2024-11-16T04:25:19,501 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:19,502 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/c6ed382680a24f0c88985eb8c44eb68b is 50, key is test_row_0/B:col10/1731731117061/Put/seqid=0 2024-11-16T04:25:19,518 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:19,519 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/28af20ad57e243b38d761de0fcb4ef1c is 50, key is test_row_0/A:col10/1731731117061/Put/seqid=0 2024-11-16T04:25:19,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742126_1302 (size=12765) 2024-11-16T04:25:19,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742127_1303 (size=12765) 2024-11-16T04:25:19,601 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/28af20ad57e243b38d761de0fcb4ef1c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/28af20ad57e243b38d761de0fcb4ef1c 2024-11-16T04:25:19,610 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into 28af20ad57e243b38d761de0fcb4ef1c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:19,610 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:19,610 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731119480; duration=0sec 2024-11-16T04:25:19,610 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:19,610 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:19,612 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:19,613 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:19,613 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:19,613 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:19,613 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f0b2871606847a582cd057daef6a7fe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/fed81d56472740a4b9002b37dba7f966, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7d6e368631294b23a602420d29c185d2] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.1 K 2024-11-16T04:25:19,614 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0b2871606847a582cd057daef6a7fe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731731114710 2024-11-16T04:25:19,614 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed81d56472740a4b9002b37dba7f966, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731731115882 2024-11-16T04:25:19,618 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6e368631294b23a602420d29c185d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731117056 2024-11-16T04:25:19,635 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:19,635 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ea986e73da8a4fed9209dd3c95230452 is 50, key is test_row_0/C:col10/1731731117061/Put/seqid=0 2024-11-16T04:25:19,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742128_1304 (size=12765) 2024-11-16T04:25:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:19,747 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:19,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T04:25:19,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7fac969936a944cfaee4c7eadc7612e0 is 50, key is test_row_0/A:col10/1731731118205/Put/seqid=0 2024-11-16T04:25:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731179781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731179782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742129_1305 (size=12251) 2024-11-16T04:25:19,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7fac969936a944cfaee4c7eadc7612e0 2024-11-16T04:25:19,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8b81b27dc6a84f0a96241f820ac36415 is 50, key is test_row_0/B:col10/1731731118205/Put/seqid=0 2024-11-16T04:25:19,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742130_1306 (size=12251) 2024-11-16T04:25:19,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8b81b27dc6a84f0a96241f820ac36415 2024-11-16T04:25:19,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/6f1937aaa8844ba9845e7656105e6ec4 is 50, key is test_row_0/C:col10/1731731118205/Put/seqid=0 2024-11-16T04:25:19,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742131_1307 (size=12251) 2024-11-16T04:25:19,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/6f1937aaa8844ba9845e7656105e6ec4 2024-11-16T04:25:19,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/7fac969936a944cfaee4c7eadc7612e0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7fac969936a944cfaee4c7eadc7612e0 2024-11-16T04:25:19,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7fac969936a944cfaee4c7eadc7612e0, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T04:25:19,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8b81b27dc6a84f0a96241f820ac36415 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8b81b27dc6a84f0a96241f820ac36415 2024-11-16T04:25:19,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8b81b27dc6a84f0a96241f820ac36415, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T04:25:19,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731179893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/6f1937aaa8844ba9845e7656105e6ec4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6f1937aaa8844ba9845e7656105e6ec4 2024-11-16T04:25:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731179894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:19,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6f1937aaa8844ba9845e7656105e6ec4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-16T04:25:19,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6a049fe61dfb22dfb7ca7a9173671d57 in 153ms, sequenceid=272, compaction requested=false 2024-11-16T04:25:19,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:19,976 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/c6ed382680a24f0c88985eb8c44eb68b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/c6ed382680a24f0c88985eb8c44eb68b 2024-11-16T04:25:19,984 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into c6ed382680a24f0c88985eb8c44eb68b(size=12.5 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:19,984 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:19,984 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731119480; duration=0sec 2024-11-16T04:25:19,985 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:19,985 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T04:25:20,044 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:20,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T04:25:20,049 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T04:25:20,050 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:20,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:20,052 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ea986e73da8a4fed9209dd3c95230452 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea986e73da8a4fed9209dd3c95230452 2024-11-16T04:25:20,059 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into ea986e73da8a4fed9209dd3c95230452(size=12.5 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:20,059 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:20,059 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731119481; duration=0sec 2024-11-16T04:25:20,059 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:20,059 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:20,106 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:20,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:25:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:20,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3a8474082c04424ab63431c22ef6e9f3 is 50, key is test_row_0/A:col10/1731731119762/Put/seqid=0 2024-11-16T04:25:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T04:25:20,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731180162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731180165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742132_1308 (size=12301) 2024-11-16T04:25:20,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T04:25:20,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731180273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731180273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T04:25:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:20,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T04:25:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731180373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,378 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731180384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,387 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731180384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,389 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731180484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731180484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T04:25:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:20,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3a8474082c04424ab63431c22ef6e9f3 2024-11-16T04:25:20,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/d0c5babab7bc4bc0bffa903068174942 is 50, key is test_row_0/B:col10/1731731119762/Put/seqid=0 2024-11-16T04:25:20,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742133_1309 (size=12301) 2024-11-16T04:25:20,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/d0c5babab7bc4bc0bffa903068174942 2024-11-16T04:25:20,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ba503c7ff8a84f6499dd37b0ebf540d3 is 50, key is test_row_0/C:col10/1731731119762/Put/seqid=0 2024-11-16T04:25:20,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742134_1310 (size=12301) 2024-11-16T04:25:20,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ba503c7ff8a84f6499dd37b0ebf540d3 2024-11-16T04:25:20,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3a8474082c04424ab63431c22ef6e9f3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3a8474082c04424ab63431c22ef6e9f3 2024-11-16T04:25:20,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3a8474082c04424ab63431c22ef6e9f3, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T04:25:20,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/d0c5babab7bc4bc0bffa903068174942 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/d0c5babab7bc4bc0bffa903068174942 2024-11-16T04:25:20,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/d0c5babab7bc4bc0bffa903068174942, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T04:25:20,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ba503c7ff8a84f6499dd37b0ebf540d3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba503c7ff8a84f6499dd37b0ebf540d3 2024-11-16T04:25:20,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba503c7ff8a84f6499dd37b0ebf540d3, entries=150, sequenceid=289, filesize=12.0 K 2024-11-16T04:25:20,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6a049fe61dfb22dfb7ca7a9173671d57 in 554ms, sequenceid=289, compaction requested=true 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:20,660 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:20,660 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:20,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:20,661 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:20,661 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:20,661 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:20,661 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:20,661 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,661 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,661 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/c6ed382680a24f0c88985eb8c44eb68b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8b81b27dc6a84f0a96241f820ac36415, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/d0c5babab7bc4bc0bffa903068174942] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.4 K 2024-11-16T04:25:20,661 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/28af20ad57e243b38d761de0fcb4ef1c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7fac969936a944cfaee4c7eadc7612e0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3a8474082c04424ab63431c22ef6e9f3] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.4 K 2024-11-16T04:25:20,662 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ed382680a24f0c88985eb8c44eb68b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731117056 2024-11-16T04:25:20,662 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28af20ad57e243b38d761de0fcb4ef1c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731117056 2024-11-16T04:25:20,662 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b81b27dc6a84f0a96241f820ac36415, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731731118205 2024-11-16T04:25:20,662 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c5babab7bc4bc0bffa903068174942, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731731119762 2024-11-16T04:25:20,663 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fac969936a944cfaee4c7eadc7612e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731731118205 2024-11-16T04:25:20,663 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a8474082c04424ab63431c22ef6e9f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731731119762 2024-11-16T04:25:20,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T04:25:20,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,665 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:25:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T04:25:20,690 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:20,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/13eeffd5ce7643bc817c142f6fda619c is 50, key is test_row_0/A:col10/1731731120162/Put/seqid=0 2024-11-16T04:25:20,690 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8a0f3ab85c40468f90c445fc19616622 is 50, key is test_row_0/B:col10/1731731119762/Put/seqid=0 2024-11-16T04:25:20,691 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:20,691 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/4b6138fca5014deea4b49e10d74f78e8 is 50, key is test_row_0/A:col10/1731731119762/Put/seqid=0 2024-11-16T04:25:20,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742135_1311 (size=12301) 2024-11-16T04:25:20,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742136_1312 (size=13017) 2024-11-16T04:25:20,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742137_1313 (size=13017) 2024-11-16T04:25:20,728 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8a0f3ab85c40468f90c445fc19616622 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8a0f3ab85c40468f90c445fc19616622 2024-11-16T04:25:20,734 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into 8a0f3ab85c40468f90c445fc19616622(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:20,734 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:20,734 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731120660; duration=0sec 2024-11-16T04:25:20,734 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:20,734 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:20,734 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:20,736 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:20,736 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:20,736 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:20,736 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea986e73da8a4fed9209dd3c95230452, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6f1937aaa8844ba9845e7656105e6ec4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba503c7ff8a84f6499dd37b0ebf540d3] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.4 K 2024-11-16T04:25:20,737 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ea986e73da8a4fed9209dd3c95230452, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731117056 2024-11-16T04:25:20,737 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1937aaa8844ba9845e7656105e6ec4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731731118205 2024-11-16T04:25:20,737 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ba503c7ff8a84f6499dd37b0ebf540d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731731119762 2024-11-16T04:25:20,749 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:20,749 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/5d0e32d8974345cabbbaddf644b691b0 is 50, key is test_row_0/C:col10/1731731119762/Put/seqid=0 2024-11-16T04:25:20,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742138_1314 (size=13017) 2024-11-16T04:25:20,784 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/5d0e32d8974345cabbbaddf644b691b0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/5d0e32d8974345cabbbaddf644b691b0 2024-11-16T04:25:20,796 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into 5d0e32d8974345cabbbaddf644b691b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:20,796 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:20,797 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731120660; duration=0sec 2024-11-16T04:25:20,797 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:20,797 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:20,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:20,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731180835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731180835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731180944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:20,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731180944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:20,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,118 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/13eeffd5ce7643bc817c142f6fda619c 2024-11-16T04:25:21,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8cc96f0babac4332a139a2034cafb16b is 50, key is test_row_0/B:col10/1731731120162/Put/seqid=0 2024-11-16T04:25:21,133 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/4b6138fca5014deea4b49e10d74f78e8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4b6138fca5014deea4b49e10d74f78e8 2024-11-16T04:25:21,140 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into 4b6138fca5014deea4b49e10d74f78e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:21,140 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:21,140 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731120660; duration=0sec 2024-11-16T04:25:21,141 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:21,141 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742139_1315 (size=12301) 2024-11-16T04:25:21,147 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8cc96f0babac4332a139a2034cafb16b 2024-11-16T04:25:21,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731181154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731181154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/c1f62965f7934283a0ed41b71998250f is 50, key is test_row_0/C:col10/1731731120162/Put/seqid=0 2024-11-16T04:25:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T04:25:21,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742140_1316 (size=12301) 2024-11-16T04:25:21,201 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/c1f62965f7934283a0ed41b71998250f 2024-11-16T04:25:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/13eeffd5ce7643bc817c142f6fda619c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/13eeffd5ce7643bc817c142f6fda619c 2024-11-16T04:25:21,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/13eeffd5ce7643bc817c142f6fda619c, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T04:25:21,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/8cc96f0babac4332a139a2034cafb16b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8cc96f0babac4332a139a2034cafb16b 2024-11-16T04:25:21,217 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8cc96f0babac4332a139a2034cafb16b, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T04:25:21,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/c1f62965f7934283a0ed41b71998250f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c1f62965f7934283a0ed41b71998250f 2024-11-16T04:25:21,227 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c1f62965f7934283a0ed41b71998250f, entries=150, sequenceid=310, filesize=12.0 K 2024-11-16T04:25:21,229 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6a049fe61dfb22dfb7ca7a9173671d57 in 563ms, sequenceid=310, compaction requested=false 2024-11-16T04:25:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T04:25:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T04:25:21,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T04:25:21,233 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1800 sec 2024-11-16T04:25:21,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.1880 sec 2024-11-16T04:25:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:21,468 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:21,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T04:25:21,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3e6dd1c481bf428bb8cfd429f59ea5cf is 50, key is test_row_0/A:col10/1731731120832/Put/seqid=0 2024-11-16T04:25:21,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742141_1317 (size=14741) 2024-11-16T04:25:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731181530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731181531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731181643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731181643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731181853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731181853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:21,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3e6dd1c481bf428bb8cfd429f59ea5cf 2024-11-16T04:25:21,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/53bf2b6563274467b9fc806673d96862 is 50, key is test_row_0/B:col10/1731731120832/Put/seqid=0 2024-11-16T04:25:21,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742142_1318 (size=12301) 2024-11-16T04:25:21,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/53bf2b6563274467b9fc806673d96862 2024-11-16T04:25:21,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/25bbbbe72be649b588c81a38577c21a8 is 50, key is test_row_0/C:col10/1731731120832/Put/seqid=0 2024-11-16T04:25:21,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742143_1319 (size=12301) 2024-11-16T04:25:22,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731182163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731182163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T04:25:22,194 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:22,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T04:25:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T04:25:22,196 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:22,197 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:22,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T04:25:22,337 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T04:25:22,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T04:25:22,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:22,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:22,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59182 deadline: 1731731182393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,396 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/25bbbbe72be649b588c81a38577c21a8 2024-11-16T04:25:22,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59134 deadline: 1731731182403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,407 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/3e6dd1c481bf428bb8cfd429f59ea5cf as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3e6dd1c481bf428bb8cfd429f59ea5cf 2024-11-16T04:25:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59148 deadline: 1731731182414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,419 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 6a049fe61dfb22dfb7ca7a9173671d57 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3e6dd1c481bf428bb8cfd429f59ea5cf, entries=200, sequenceid=329, filesize=14.4 K 2024-11-16T04:25:22,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/53bf2b6563274467b9fc806673d96862 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/53bf2b6563274467b9fc806673d96862 2024-11-16T04:25:22,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/53bf2b6563274467b9fc806673d96862, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T04:25:22,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/25bbbbe72be649b588c81a38577c21a8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/25bbbbe72be649b588c81a38577c21a8 2024-11-16T04:25:22,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/25bbbbe72be649b588c81a38577c21a8, entries=150, sequenceid=329, filesize=12.0 K 2024-11-16T04:25:22,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6a049fe61dfb22dfb7ca7a9173671d57 in 967ms, sequenceid=329, compaction requested=true 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:22,436 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:22,436 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:22,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:22,437 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:22,437 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:22,437 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:22,438 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4b6138fca5014deea4b49e10d74f78e8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/13eeffd5ce7643bc817c142f6fda619c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3e6dd1c481bf428bb8cfd429f59ea5cf] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=39.1 K 2024-11-16T04:25:22,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:22,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:22,438 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:22,438 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b6138fca5014deea4b49e10d74f78e8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731731119762 2024-11-16T04:25:22,438 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8a0f3ab85c40468f90c445fc19616622, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8cc96f0babac4332a139a2034cafb16b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/53bf2b6563274467b9fc806673d96862] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.7 K 2024-11-16T04:25:22,438 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13eeffd5ce7643bc817c142f6fda619c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731731120150 2024-11-16T04:25:22,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0f3ab85c40468f90c445fc19616622, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731731119762 2024-11-16T04:25:22,439 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e6dd1c481bf428bb8cfd429f59ea5cf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731120811 2024-11-16T04:25:22,439 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc96f0babac4332a139a2034cafb16b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731731120150 2024-11-16T04:25:22,439 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 53bf2b6563274467b9fc806673d96862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731120811 2024-11-16T04:25:22,450 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:22,451 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/19dc0ede5fc44f919af0486d9b9f7cf7 is 50, key is test_row_0/A:col10/1731731120832/Put/seqid=0 2024-11-16T04:25:22,451 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:22,452 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/f40ca0adc9cd4e7db57b34bea86806f0 is 50, key is test_row_0/B:col10/1731731120832/Put/seqid=0 2024-11-16T04:25:22,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742144_1320 (size=13119) 2024-11-16T04:25:22,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T04:25:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:22,506 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-16T04:25:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T04:25:22,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742145_1321 (size=13119) 2024-11-16T04:25:22,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/b9bebabcdfd24a09a630419aabdbd55b is 50, key is test_row_0/A:col10/1731731121529/Put/seqid=0 2024-11-16T04:25:22,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742146_1322 (size=12301) 2024-11-16T04:25:22,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. as already flushing 2024-11-16T04:25:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:22,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731182738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731182739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T04:25:22,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731182853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731182853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:22,910 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/19dc0ede5fc44f919af0486d9b9f7cf7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/19dc0ede5fc44f919af0486d9b9f7cf7 2024-11-16T04:25:22,917 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into 19dc0ede5fc44f919af0486d9b9f7cf7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:22,917 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:22,917 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731122436; duration=0sec 2024-11-16T04:25:22,918 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:22,918 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:22,918 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:22,919 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:22,919 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:22,919 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:22,919 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/5d0e32d8974345cabbbaddf644b691b0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c1f62965f7934283a0ed41b71998250f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/25bbbbe72be649b588c81a38577c21a8] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.7 K 2024-11-16T04:25:22,920 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d0e32d8974345cabbbaddf644b691b0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731731119762 2024-11-16T04:25:22,920 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f62965f7934283a0ed41b71998250f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731731120150 2024-11-16T04:25:22,921 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25bbbbe72be649b588c81a38577c21a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731120811 2024-11-16T04:25:22,931 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/f40ca0adc9cd4e7db57b34bea86806f0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/f40ca0adc9cd4e7db57b34bea86806f0 2024-11-16T04:25:22,936 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into f40ca0adc9cd4e7db57b34bea86806f0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:22,936 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:22,936 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731122436; duration=0sec 2024-11-16T04:25:22,937 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:22,937 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:22,942 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:22,942 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/20ce69a3e9e9446a9d35d3b686611f3c is 50, key is test_row_0/C:col10/1731731120832/Put/seqid=0 2024-11-16T04:25:22,987 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/b9bebabcdfd24a09a630419aabdbd55b 2024-11-16T04:25:23,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/2ae24effafaf49a99e94abd20bac7fcd is 50, key is test_row_0/B:col10/1731731121529/Put/seqid=0 2024-11-16T04:25:23,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742147_1323 (size=13119) 2024-11-16T04:25:23,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742148_1324 (size=12301) 2024-11-16T04:25:23,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731183063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:23,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:23,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731183073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T04:25:23,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731183373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:23,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731183383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:23,426 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/20ce69a3e9e9446a9d35d3b686611f3c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/20ce69a3e9e9446a9d35d3b686611f3c 2024-11-16T04:25:23,443 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into 20ce69a3e9e9446a9d35d3b686611f3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:23,443 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:23,443 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731122436; duration=0sec 2024-11-16T04:25:23,443 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:23,443 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:23,452 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/2ae24effafaf49a99e94abd20bac7fcd 2024-11-16T04:25:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/475c97138c804fe2a463ca0add47d70a is 50, key is test_row_0/C:col10/1731731121529/Put/seqid=0 2024-11-16T04:25:23,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742149_1325 (size=12301) 2024-11-16T04:25:23,495 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/475c97138c804fe2a463ca0add47d70a 2024-11-16T04:25:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/b9bebabcdfd24a09a630419aabdbd55b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b9bebabcdfd24a09a630419aabdbd55b 2024-11-16T04:25:23,503 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b9bebabcdfd24a09a630419aabdbd55b, entries=150, sequenceid=349, filesize=12.0 K 2024-11-16T04:25:23,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/2ae24effafaf49a99e94abd20bac7fcd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/2ae24effafaf49a99e94abd20bac7fcd 2024-11-16T04:25:23,510 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/2ae24effafaf49a99e94abd20bac7fcd, entries=150, sequenceid=349, filesize=12.0 K 2024-11-16T04:25:23,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/475c97138c804fe2a463ca0add47d70a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/475c97138c804fe2a463ca0add47d70a 2024-11-16T04:25:23,519 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/475c97138c804fe2a463ca0add47d70a, entries=150, sequenceid=349, filesize=12.0 K 2024-11-16T04:25:23,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 6a049fe61dfb22dfb7ca7a9173671d57 in 1015ms, sequenceid=349, compaction requested=false 2024-11-16T04:25:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:23,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T04:25:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T04:25:23,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T04:25:23,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3240 sec 2024-11-16T04:25:23,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.3310 sec 2024-11-16T04:25:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:23,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-16T04:25:23,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/0de56421f25141e7af50bf7b08f07b4e is 50, key is test_row_0/A:col10/1731731122730/Put/seqid=0 2024-11-16T04:25:23,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742150_1326 (size=14741) 2024-11-16T04:25:23,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/0de56421f25141e7af50bf7b08f07b4e 2024-11-16T04:25:23,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731183951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:23,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/1dedf3eea0274c68a691e5048c4286e7 is 50, key is test_row_0/B:col10/1731731122730/Put/seqid=0 2024-11-16T04:25:23,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:23,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731183955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:23,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:24,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742151_1327 (size=12301) 2024-11-16T04:25:24,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:24,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731184063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:24,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731184074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:24,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:24,127 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-11-16T04:25:24,127 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,127 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,127 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,132 INFO [Thread-1164 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1164. 2024-11-16T04:25:24,132 DEBUG [Thread-1164 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,132 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,132 INFO [Thread-1166 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1166. 2024-11-16T04:25:24,132 DEBUG [Thread-1166 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,132 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,132 DEBUG [Thread-1166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,132 DEBUG [Thread-1164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,134 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-16T04:25:24,134 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,134 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,135 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,136 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-11-16T04:25:24,136 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,136 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,136 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59132 deadline: 1731731184273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:24,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:24,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59192 deadline: 1731731184284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:24,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T04:25:24,344 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:24,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/1dedf3eea0274c68a691e5048c4286e7 2024-11-16T04:25:24,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7f00b9adb0d34d5892b25fb9fbcbf1b3 is 50, key is test_row_0/C:col10/1731731122730/Put/seqid=0 2024-11-16T04:25:24,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742152_1328 (size=12301) 2024-11-16T04:25:24,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7f00b9adb0d34d5892b25fb9fbcbf1b3 2024-11-16T04:25:24,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/0de56421f25141e7af50bf7b08f07b4e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/0de56421f25141e7af50bf7b08f07b4e 2024-11-16T04:25:24,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/0de56421f25141e7af50bf7b08f07b4e, entries=200, sequenceid=370, filesize=14.4 K 2024-11-16T04:25:24,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/1dedf3eea0274c68a691e5048c4286e7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/1dedf3eea0274c68a691e5048c4286e7 2024-11-16T04:25:24,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/1dedf3eea0274c68a691e5048c4286e7, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T04:25:24,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/7f00b9adb0d34d5892b25fb9fbcbf1b3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f00b9adb0d34d5892b25fb9fbcbf1b3 2024-11-16T04:25:24,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f00b9adb0d34d5892b25fb9fbcbf1b3, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T04:25:24,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 6a049fe61dfb22dfb7ca7a9173671d57 in 580ms, sequenceid=370, compaction requested=true 2024-11-16T04:25:24,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:24,469 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:24,470 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6a049fe61dfb22dfb7ca7a9173671d57:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:24,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:24,471 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:24,471 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/A is initiating minor compaction (all files) 2024-11-16T04:25:24,471 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/A in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:24,472 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/19dc0ede5fc44f919af0486d9b9f7cf7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b9bebabcdfd24a09a630419aabdbd55b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/0de56421f25141e7af50bf7b08f07b4e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=39.2 K 2024-11-16T04:25:24,472 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:24,472 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/B is initiating minor compaction (all files) 2024-11-16T04:25:24,472 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/B in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:24,473 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/f40ca0adc9cd4e7db57b34bea86806f0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/2ae24effafaf49a99e94abd20bac7fcd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/1dedf3eea0274c68a691e5048c4286e7] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.8 K 2024-11-16T04:25:24,473 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19dc0ede5fc44f919af0486d9b9f7cf7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731120811 2024-11-16T04:25:24,473 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f40ca0adc9cd4e7db57b34bea86806f0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731120811 2024-11-16T04:25:24,473 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9bebabcdfd24a09a630419aabdbd55b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731731121520 2024-11-16T04:25:24,473 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de56421f25141e7af50bf7b08f07b4e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731731122700 2024-11-16T04:25:24,473 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ae24effafaf49a99e94abd20bac7fcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731731121520 2024-11-16T04:25:24,474 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dedf3eea0274c68a691e5048c4286e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731731122700 2024-11-16T04:25:24,498 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#A#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:24,498 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/4f33f40f58ce401abd7778e603c15081 is 50, key is test_row_0/A:col10/1731731122730/Put/seqid=0 2024-11-16T04:25:24,506 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:24,506 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6fc2ddf3386646f4bcd35d6a89d9696a is 50, key is test_row_0/B:col10/1731731122730/Put/seqid=0 2024-11-16T04:25:24,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742153_1329 (size=13221) 2024-11-16T04:25:24,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742154_1330 (size=13221) 2024-11-16T04:25:24,530 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/6fc2ddf3386646f4bcd35d6a89d9696a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6fc2ddf3386646f4bcd35d6a89d9696a 2024-11-16T04:25:24,538 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/B of 6a049fe61dfb22dfb7ca7a9173671d57 into 6fc2ddf3386646f4bcd35d6a89d9696a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:24,538 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:24,538 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/B, priority=13, startTime=1731731124470; duration=0sec 2024-11-16T04:25:24,538 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:24,538 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:B 2024-11-16T04:25:24,538 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:24,539 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:24,539 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 6a049fe61dfb22dfb7ca7a9173671d57/C is initiating minor compaction (all files) 2024-11-16T04:25:24,539 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6a049fe61dfb22dfb7ca7a9173671d57/C in TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:24,540 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/20ce69a3e9e9446a9d35d3b686611f3c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/475c97138c804fe2a463ca0add47d70a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f00b9adb0d34d5892b25fb9fbcbf1b3] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp, totalSize=36.8 K 2024-11-16T04:25:24,542 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 20ce69a3e9e9446a9d35d3b686611f3c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731731120811 2024-11-16T04:25:24,543 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 475c97138c804fe2a463ca0add47d70a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731731121520 2024-11-16T04:25:24,543 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f00b9adb0d34d5892b25fb9fbcbf1b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731731122700 2024-11-16T04:25:24,550 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6a049fe61dfb22dfb7ca7a9173671d57#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:24,550 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/56dd3effadf94bfcaf6ac05d566107c2 is 50, key is test_row_0/C:col10/1731731122730/Put/seqid=0 2024-11-16T04:25:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742155_1331 (size=13221) 2024-11-16T04:25:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:24,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-16T04:25:24,585 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-16T04:25:24,585 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,585 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,586 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/1471cc9cbaa24e7a99bc1ac2398d7125 is 50, key is test_row_0/A:col10/1731731123949/Put/seqid=0 2024-11-16T04:25:24,596 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-16T04:25:24,596 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:24,596 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,596 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:24,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:24,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742156_1332 (size=12301) 2024-11-16T04:25:24,912 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/4f33f40f58ce401abd7778e603c15081 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4f33f40f58ce401abd7778e603c15081 2024-11-16T04:25:24,916 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/A of 6a049fe61dfb22dfb7ca7a9173671d57 into 4f33f40f58ce401abd7778e603c15081(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:24,916 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:24,916 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/A, priority=13, startTime=1731731124469; duration=0sec 2024-11-16T04:25:24,916 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:24,916 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:A 2024-11-16T04:25:24,966 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/56dd3effadf94bfcaf6ac05d566107c2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/56dd3effadf94bfcaf6ac05d566107c2 2024-11-16T04:25:24,976 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6a049fe61dfb22dfb7ca7a9173671d57/C of 6a049fe61dfb22dfb7ca7a9173671d57 into 56dd3effadf94bfcaf6ac05d566107c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:24,976 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:24,976 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., storeName=6a049fe61dfb22dfb7ca7a9173671d57/C, priority=13, startTime=1731731124470; duration=0sec 2024-11-16T04:25:24,976 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:24,976 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6a049fe61dfb22dfb7ca7a9173671d57:C 2024-11-16T04:25:25,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/1471cc9cbaa24e7a99bc1ac2398d7125 2024-11-16T04:25:25,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/b61f0f3071154efaaf95ad2be8b2094a is 50, key is test_row_0/B:col10/1731731123949/Put/seqid=0 2024-11-16T04:25:25,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742157_1333 (size=12301) 2024-11-16T04:25:25,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/b61f0f3071154efaaf95ad2be8b2094a 2024-11-16T04:25:25,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ba1bfc7f72aa429fb089ed307f5839b4 is 50, key is test_row_0/C:col10/1731731123949/Put/seqid=0 2024-11-16T04:25:25,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742158_1334 (size=12301) 2024-11-16T04:25:25,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ba1bfc7f72aa429fb089ed307f5839b4 2024-11-16T04:25:25,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/1471cc9cbaa24e7a99bc1ac2398d7125 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1471cc9cbaa24e7a99bc1ac2398d7125 2024-11-16T04:25:25,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1471cc9cbaa24e7a99bc1ac2398d7125, entries=150, sequenceid=390, filesize=12.0 K 2024-11-16T04:25:25,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/b61f0f3071154efaaf95ad2be8b2094a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/b61f0f3071154efaaf95ad2be8b2094a 2024-11-16T04:25:25,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/b61f0f3071154efaaf95ad2be8b2094a, entries=150, sequenceid=390, filesize=12.0 K 2024-11-16T04:25:25,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/ba1bfc7f72aa429fb089ed307f5839b4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba1bfc7f72aa429fb089ed307f5839b4 2024-11-16T04:25:25,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba1bfc7f72aa429fb089ed307f5839b4, entries=150, sequenceid=390, filesize=12.0 K 2024-11-16T04:25:25,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=6.71 KB/6870 for 6a049fe61dfb22dfb7ca7a9173671d57 in 882ms, sequenceid=390, compaction requested=false 2024-11-16T04:25:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6a049fe61dfb22dfb7ca7a9173671d57: 2024-11-16T04:25:26,435 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-11-16T04:25:26,435 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:26,435 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,435 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:26,435 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-16T04:25:26,435 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:26,435 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,435 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:26,445 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-16T04:25:26,445 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:26,445 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,445 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3722 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3663 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3630 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3702 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3659 2024-11-16T04:25:26,445 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T04:25:26,446 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:25:26,446 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:25:26,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:26,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:26,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T04:25:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T04:25:26,450 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731126450"}]},"ts":"1731731126450"} 2024-11-16T04:25:26,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T04:25:26,451 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T04:25:26,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:25:26,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, UNASSIGN}] 2024-11-16T04:25:26,454 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, UNASSIGN 2024-11-16T04:25:26,455 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=6a049fe61dfb22dfb7ca7a9173671d57, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:26,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, UNASSIGN because future has completed 2024-11-16T04:25:26,456 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:25:26,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T04:25:26,610 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:26,610 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:25:26,610 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 6a049fe61dfb22dfb7ca7a9173671d57, disabling compactions & flushes 2024-11-16T04:25:26,610 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:26,610 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:26,610 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. after waiting 0 ms 2024-11-16T04:25:26,610 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:26,610 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 6a049fe61dfb22dfb7ca7a9173671d57 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T04:25:26,615 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/83a0e08874cd4e8d87af26c42d92b05b is 50, key is test_row_0/A:col10/1731731126444/Put/seqid=0 2024-11-16T04:25:26,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742159_1335 (size=12301) 2024-11-16T04:25:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T04:25:27,020 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/83a0e08874cd4e8d87af26c42d92b05b 2024-11-16T04:25:27,027 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/294e861a72284be0ac4691b6c5bf10d3 is 50, key is test_row_0/B:col10/1731731126444/Put/seqid=0 2024-11-16T04:25:27,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742160_1336 (size=12301) 2024-11-16T04:25:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T04:25:27,432 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/294e861a72284be0ac4691b6c5bf10d3 2024-11-16T04:25:27,438 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/f630cb196822475eba2475b962f8e907 is 50, key is test_row_0/C:col10/1731731126444/Put/seqid=0 2024-11-16T04:25:27,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742161_1337 (size=12301) 2024-11-16T04:25:27,442 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/f630cb196822475eba2475b962f8e907 2024-11-16T04:25:27,447 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/A/83a0e08874cd4e8d87af26c42d92b05b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/83a0e08874cd4e8d87af26c42d92b05b 2024-11-16T04:25:27,451 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/83a0e08874cd4e8d87af26c42d92b05b, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T04:25:27,452 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/B/294e861a72284be0ac4691b6c5bf10d3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/294e861a72284be0ac4691b6c5bf10d3 2024-11-16T04:25:27,456 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/294e861a72284be0ac4691b6c5bf10d3, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T04:25:27,457 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/.tmp/C/f630cb196822475eba2475b962f8e907 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f630cb196822475eba2475b962f8e907 2024-11-16T04:25:27,461 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f630cb196822475eba2475b962f8e907, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T04:25:27,462 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6a049fe61dfb22dfb7ca7a9173671d57 in 852ms, sequenceid=399, compaction requested=true 2024-11-16T04:25:27,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7a0fd48c9e25409d94582ac55f72ca1c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/524f2bf5f9714761b1c79ffb0fe18569, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7c40ffdc63bd4ff799ed0c6ca8db1da2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/fd07d775dd354604a74fc942e1c6e1bb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b403b72d83e54ce1a8cdc1f2a4764c3d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3c32d082997f42c096ce025d19f22290, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1f594bbcb56c45599387e99e4b0768b4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/04b392fc224a4a79814a709d7dca1636, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/e7d107e8609e42e0b4e8dbea2d8e9a2b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7e0733a5d974471a6724309f44bdf94, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/5548c3dab564469984dfc331cda54319, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/442fe06680a04cd4b3fafec3975651a3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/77c8d99710704ab09863e38f649907e8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/29bface4c73c4a04892bced0e2e746fd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/a30e3eff179d401bbad2b90c9d453192, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/18ed3187a6a2428ba645d8f30f0cf8d3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3eb8e5137d384e41850ade9084bd7105, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/28af20ad57e243b38d761de0fcb4ef1c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7fac969936a944cfaee4c7eadc7612e0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4b6138fca5014deea4b49e10d74f78e8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3a8474082c04424ab63431c22ef6e9f3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/13eeffd5ce7643bc817c142f6fda619c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3e6dd1c481bf428bb8cfd429f59ea5cf, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/19dc0ede5fc44f919af0486d9b9f7cf7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b9bebabcdfd24a09a630419aabdbd55b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/0de56421f25141e7af50bf7b08f07b4e] to archive 2024-11-16T04:25:27,464 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:27,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7a0fd48c9e25409d94582ac55f72ca1c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7a0fd48c9e25409d94582ac55f72ca1c 2024-11-16T04:25:27,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/524f2bf5f9714761b1c79ffb0fe18569 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/524f2bf5f9714761b1c79ffb0fe18569 2024-11-16T04:25:27,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7c40ffdc63bd4ff799ed0c6ca8db1da2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7c40ffdc63bd4ff799ed0c6ca8db1da2 2024-11-16T04:25:27,468 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/fd07d775dd354604a74fc942e1c6e1bb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/fd07d775dd354604a74fc942e1c6e1bb 2024-11-16T04:25:27,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b403b72d83e54ce1a8cdc1f2a4764c3d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b403b72d83e54ce1a8cdc1f2a4764c3d 2024-11-16T04:25:27,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3c32d082997f42c096ce025d19f22290 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3c32d082997f42c096ce025d19f22290 2024-11-16T04:25:27,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1f594bbcb56c45599387e99e4b0768b4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1f594bbcb56c45599387e99e4b0768b4 2024-11-16T04:25:27,472 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/04b392fc224a4a79814a709d7dca1636 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/04b392fc224a4a79814a709d7dca1636 2024-11-16T04:25:27,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/e7d107e8609e42e0b4e8dbea2d8e9a2b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/e7d107e8609e42e0b4e8dbea2d8e9a2b 2024-11-16T04:25:27,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7e0733a5d974471a6724309f44bdf94 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7e0733a5d974471a6724309f44bdf94 2024-11-16T04:25:27,477 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/5548c3dab564469984dfc331cda54319 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/5548c3dab564469984dfc331cda54319 2024-11-16T04:25:27,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/442fe06680a04cd4b3fafec3975651a3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/442fe06680a04cd4b3fafec3975651a3 2024-11-16T04:25:27,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/77c8d99710704ab09863e38f649907e8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/77c8d99710704ab09863e38f649907e8 2024-11-16T04:25:27,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/d7cd7bc064d34ba8bb93d9b91b0c2d9e 2024-11-16T04:25:27,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/29bface4c73c4a04892bced0e2e746fd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/29bface4c73c4a04892bced0e2e746fd 2024-11-16T04:25:27,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/a30e3eff179d401bbad2b90c9d453192 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/a30e3eff179d401bbad2b90c9d453192 2024-11-16T04:25:27,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/18ed3187a6a2428ba645d8f30f0cf8d3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/18ed3187a6a2428ba645d8f30f0cf8d3 2024-11-16T04:25:27,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3eb8e5137d384e41850ade9084bd7105 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3eb8e5137d384e41850ade9084bd7105 2024-11-16T04:25:27,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/28af20ad57e243b38d761de0fcb4ef1c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/28af20ad57e243b38d761de0fcb4ef1c 2024-11-16T04:25:27,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7fac969936a944cfaee4c7eadc7612e0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/7fac969936a944cfaee4c7eadc7612e0 2024-11-16T04:25:27,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4b6138fca5014deea4b49e10d74f78e8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4b6138fca5014deea4b49e10d74f78e8 2024-11-16T04:25:27,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3a8474082c04424ab63431c22ef6e9f3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3a8474082c04424ab63431c22ef6e9f3 2024-11-16T04:25:27,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/13eeffd5ce7643bc817c142f6fda619c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/13eeffd5ce7643bc817c142f6fda619c 2024-11-16T04:25:27,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3e6dd1c481bf428bb8cfd429f59ea5cf to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/3e6dd1c481bf428bb8cfd429f59ea5cf 2024-11-16T04:25:27,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/19dc0ede5fc44f919af0486d9b9f7cf7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/19dc0ede5fc44f919af0486d9b9f7cf7 2024-11-16T04:25:27,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b9bebabcdfd24a09a630419aabdbd55b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/b9bebabcdfd24a09a630419aabdbd55b 2024-11-16T04:25:27,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/0de56421f25141e7af50bf7b08f07b4e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/0de56421f25141e7af50bf7b08f07b4e 2024-11-16T04:25:27,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6be1773ea52143fba5fc86771c6e9345, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/65b4b2de9fa942c2875239459ab53b35, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/63c2b5dd961c43de8c16e746b499b98c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/aede92296500453fbb771ab200f5831a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4ee440c0e9cd4dd0879902616511c947, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/0726916680a64fcd9adf05612cb87e90, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/9ad868c69d384b9f9419a01e8ae3db71, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4a242c83cdf6415a9d7d0c3e8620ae7e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/17c12ec1a6c44d4a996f4a1981e0ecda, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/efd406a4048e4309845a5113499946e5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/28ba1a6ca54445c28eeb010f7962ec03, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/fa1107e4e6b84969ad4ca9e118a034bb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/275d3736d08a44f0aa5b4fdb383e0294, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/bc57d90d4b164697924f2956f0e496f7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/5c31f02845364a4bbae86d7b38604b67, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6ac801baa07f48bbae5fb3d6991e2641, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/7e14344ddb454e2791f1d745a6914ef5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/c6ed382680a24f0c88985eb8c44eb68b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/284bd80fd1d3420aa28cecff3fce8298, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8b81b27dc6a84f0a96241f820ac36415, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8a0f3ab85c40468f90c445fc19616622, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/d0c5babab7bc4bc0bffa903068174942, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8cc96f0babac4332a139a2034cafb16b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/f40ca0adc9cd4e7db57b34bea86806f0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/53bf2b6563274467b9fc806673d96862, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/2ae24effafaf49a99e94abd20bac7fcd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/1dedf3eea0274c68a691e5048c4286e7] to archive 2024-11-16T04:25:27,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:27,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6be1773ea52143fba5fc86771c6e9345 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6be1773ea52143fba5fc86771c6e9345 2024-11-16T04:25:27,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/65b4b2de9fa942c2875239459ab53b35 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/65b4b2de9fa942c2875239459ab53b35 2024-11-16T04:25:27,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/63c2b5dd961c43de8c16e746b499b98c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/63c2b5dd961c43de8c16e746b499b98c 2024-11-16T04:25:27,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/aede92296500453fbb771ab200f5831a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/aede92296500453fbb771ab200f5831a 2024-11-16T04:25:27,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4ee440c0e9cd4dd0879902616511c947 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4ee440c0e9cd4dd0879902616511c947 2024-11-16T04:25:27,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/0726916680a64fcd9adf05612cb87e90 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/0726916680a64fcd9adf05612cb87e90 2024-11-16T04:25:27,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/9ad868c69d384b9f9419a01e8ae3db71 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/9ad868c69d384b9f9419a01e8ae3db71 2024-11-16T04:25:27,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4a242c83cdf6415a9d7d0c3e8620ae7e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/4a242c83cdf6415a9d7d0c3e8620ae7e 2024-11-16T04:25:27,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/17c12ec1a6c44d4a996f4a1981e0ecda to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/17c12ec1a6c44d4a996f4a1981e0ecda 2024-11-16T04:25:27,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/efd406a4048e4309845a5113499946e5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/efd406a4048e4309845a5113499946e5 2024-11-16T04:25:27,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/28ba1a6ca54445c28eeb010f7962ec03 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/28ba1a6ca54445c28eeb010f7962ec03 2024-11-16T04:25:27,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/fa1107e4e6b84969ad4ca9e118a034bb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/fa1107e4e6b84969ad4ca9e118a034bb 2024-11-16T04:25:27,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/275d3736d08a44f0aa5b4fdb383e0294 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/275d3736d08a44f0aa5b4fdb383e0294 2024-11-16T04:25:27,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/bc57d90d4b164697924f2956f0e496f7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/bc57d90d4b164697924f2956f0e496f7 2024-11-16T04:25:27,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/5c31f02845364a4bbae86d7b38604b67 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/5c31f02845364a4bbae86d7b38604b67 2024-11-16T04:25:27,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6ac801baa07f48bbae5fb3d6991e2641 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6ac801baa07f48bbae5fb3d6991e2641 2024-11-16T04:25:27,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/7e14344ddb454e2791f1d745a6914ef5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/7e14344ddb454e2791f1d745a6914ef5 2024-11-16T04:25:27,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/c6ed382680a24f0c88985eb8c44eb68b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/c6ed382680a24f0c88985eb8c44eb68b 2024-11-16T04:25:27,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/284bd80fd1d3420aa28cecff3fce8298 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/284bd80fd1d3420aa28cecff3fce8298 2024-11-16T04:25:27,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8b81b27dc6a84f0a96241f820ac36415 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8b81b27dc6a84f0a96241f820ac36415 2024-11-16T04:25:27,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8a0f3ab85c40468f90c445fc19616622 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8a0f3ab85c40468f90c445fc19616622 2024-11-16T04:25:27,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/d0c5babab7bc4bc0bffa903068174942 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/d0c5babab7bc4bc0bffa903068174942 2024-11-16T04:25:27,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8cc96f0babac4332a139a2034cafb16b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/8cc96f0babac4332a139a2034cafb16b 2024-11-16T04:25:27,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/f40ca0adc9cd4e7db57b34bea86806f0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/f40ca0adc9cd4e7db57b34bea86806f0 2024-11-16T04:25:27,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/53bf2b6563274467b9fc806673d96862 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/53bf2b6563274467b9fc806673d96862 2024-11-16T04:25:27,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/2ae24effafaf49a99e94abd20bac7fcd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/2ae24effafaf49a99e94abd20bac7fcd 2024-11-16T04:25:27,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/1dedf3eea0274c68a691e5048c4286e7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/1dedf3eea0274c68a691e5048c4286e7 2024-11-16T04:25:27,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/745ec44c4a3b4f4aba2b556a74bcafe0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6ea311a5eabe41d586ee24be561aed8b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/cdbf06a75ea4469abb147dde3dff3b0b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/e20d87cb94494aec961fa79f1568c32a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ed5f8e46bbed41e793b742fd09b08530, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7fb40d106f214974bcf606640e84b7e1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c9711d7c65a14cca89ae3819d26b23fa, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/363fdd88923240bab68b902bf99b7eba, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/08a558bea2f948939003192719369b6a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/b934f055fb9d47b994d58285687818ef, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea76fc96f8b645128169ab5d4b6d5532, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f50a89c8080045e9926ab933acc1967e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/51ed6410a82e48c298d43aa5117e2c7a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/256a31ef51fd4ee5ac9c97cf3501fb66, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f0b2871606847a582cd057daef6a7fe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/4dc5279198a042e49a34d205b1d84a71, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/fed81d56472740a4b9002b37dba7f966, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea986e73da8a4fed9209dd3c95230452, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7d6e368631294b23a602420d29c185d2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6f1937aaa8844ba9845e7656105e6ec4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/5d0e32d8974345cabbbaddf644b691b0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba503c7ff8a84f6499dd37b0ebf540d3, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c1f62965f7934283a0ed41b71998250f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/20ce69a3e9e9446a9d35d3b686611f3c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/25bbbbe72be649b588c81a38577c21a8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/475c97138c804fe2a463ca0add47d70a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f00b9adb0d34d5892b25fb9fbcbf1b3] to archive 2024-11-16T04:25:27,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:27,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/745ec44c4a3b4f4aba2b556a74bcafe0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/745ec44c4a3b4f4aba2b556a74bcafe0 2024-11-16T04:25:27,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6ea311a5eabe41d586ee24be561aed8b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6ea311a5eabe41d586ee24be561aed8b 2024-11-16T04:25:27,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/cdbf06a75ea4469abb147dde3dff3b0b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/cdbf06a75ea4469abb147dde3dff3b0b 2024-11-16T04:25:27,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/e20d87cb94494aec961fa79f1568c32a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/e20d87cb94494aec961fa79f1568c32a 2024-11-16T04:25:27,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ed5f8e46bbed41e793b742fd09b08530 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ed5f8e46bbed41e793b742fd09b08530 2024-11-16T04:25:27,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7fb40d106f214974bcf606640e84b7e1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7fb40d106f214974bcf606640e84b7e1 2024-11-16T04:25:27,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c9711d7c65a14cca89ae3819d26b23fa to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c9711d7c65a14cca89ae3819d26b23fa 2024-11-16T04:25:27,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/363fdd88923240bab68b902bf99b7eba to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/363fdd88923240bab68b902bf99b7eba 2024-11-16T04:25:27,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/08a558bea2f948939003192719369b6a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/08a558bea2f948939003192719369b6a 2024-11-16T04:25:27,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/b934f055fb9d47b994d58285687818ef to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/b934f055fb9d47b994d58285687818ef 2024-11-16T04:25:27,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea76fc96f8b645128169ab5d4b6d5532 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea76fc96f8b645128169ab5d4b6d5532 2024-11-16T04:25:27,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f50a89c8080045e9926ab933acc1967e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f50a89c8080045e9926ab933acc1967e 2024-11-16T04:25:27,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/51ed6410a82e48c298d43aa5117e2c7a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/51ed6410a82e48c298d43aa5117e2c7a 2024-11-16T04:25:27,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/256a31ef51fd4ee5ac9c97cf3501fb66 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/256a31ef51fd4ee5ac9c97cf3501fb66 2024-11-16T04:25:27,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f0b2871606847a582cd057daef6a7fe to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f0b2871606847a582cd057daef6a7fe 2024-11-16T04:25:27,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/4dc5279198a042e49a34d205b1d84a71 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/4dc5279198a042e49a34d205b1d84a71 2024-11-16T04:25:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T04:25:27,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/fed81d56472740a4b9002b37dba7f966 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/fed81d56472740a4b9002b37dba7f966 2024-11-16T04:25:27,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea986e73da8a4fed9209dd3c95230452 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ea986e73da8a4fed9209dd3c95230452 2024-11-16T04:25:27,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7d6e368631294b23a602420d29c185d2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7d6e368631294b23a602420d29c185d2 2024-11-16T04:25:27,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6f1937aaa8844ba9845e7656105e6ec4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/6f1937aaa8844ba9845e7656105e6ec4 2024-11-16T04:25:27,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/5d0e32d8974345cabbbaddf644b691b0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/5d0e32d8974345cabbbaddf644b691b0 2024-11-16T04:25:27,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba503c7ff8a84f6499dd37b0ebf540d3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba503c7ff8a84f6499dd37b0ebf540d3 2024-11-16T04:25:27,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c1f62965f7934283a0ed41b71998250f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/c1f62965f7934283a0ed41b71998250f 2024-11-16T04:25:27,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/20ce69a3e9e9446a9d35d3b686611f3c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/20ce69a3e9e9446a9d35d3b686611f3c 2024-11-16T04:25:27,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/25bbbbe72be649b588c81a38577c21a8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/25bbbbe72be649b588c81a38577c21a8 2024-11-16T04:25:27,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/475c97138c804fe2a463ca0add47d70a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/475c97138c804fe2a463ca0add47d70a 2024-11-16T04:25:27,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f00b9adb0d34d5892b25fb9fbcbf1b3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/7f00b9adb0d34d5892b25fb9fbcbf1b3 2024-11-16T04:25:27,611 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/recovered.edits/402.seqid, newMaxSeqId=402, maxSeqId=1 2024-11-16T04:25:27,612 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. 2024-11-16T04:25:27,612 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 6a049fe61dfb22dfb7ca7a9173671d57: Waiting for close lock at 1731731126610Running coprocessor pre-close hooks at 1731731126610Disabling compacts and flushes for region at 1731731126610Disabling writes for close at 1731731126610Obtaining lock to block concurrent updates at 1731731126610Preparing flush snapshotting stores in 6a049fe61dfb22dfb7ca7a9173671d57 at 1731731126610Finished memstore snapshotting TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731731126610Flushing stores of TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57. at 1731731126611 (+1 ms)Flushing 6a049fe61dfb22dfb7ca7a9173671d57/A: creating writer at 1731731126611Flushing 6a049fe61dfb22dfb7ca7a9173671d57/A: appending metadata at 1731731126614 (+3 ms)Flushing 6a049fe61dfb22dfb7ca7a9173671d57/A: closing flushed file at 1731731126614Flushing 6a049fe61dfb22dfb7ca7a9173671d57/B: creating writer at 1731731127024 (+410 ms)Flushing 6a049fe61dfb22dfb7ca7a9173671d57/B: appending metadata at 1731731127027 (+3 ms)Flushing 6a049fe61dfb22dfb7ca7a9173671d57/B: closing flushed file at 1731731127027Flushing 6a049fe61dfb22dfb7ca7a9173671d57/C: creating writer at 1731731127435 (+408 ms)Flushing 6a049fe61dfb22dfb7ca7a9173671d57/C: appending metadata at 1731731127438 (+3 ms)Flushing 6a049fe61dfb22dfb7ca7a9173671d57/C: closing flushed file at 1731731127438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15acae32: reopening flushed file at 1731731127446 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@726d65d5: reopening flushed file at 1731731127451 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55c800f5: reopening flushed file at 1731731127457 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6a049fe61dfb22dfb7ca7a9173671d57 in 852ms, sequenceid=399, compaction requested=true at 1731731127462 (+5 ms)Writing region close event to WAL at 1731731127609 (+147 ms)Running coprocessor post-close hooks at 1731731127612 (+3 ms)Closed at 1731731127612 2024-11-16T04:25:27,614 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:27,615 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=6a049fe61dfb22dfb7ca7a9173671d57, regionState=CLOSED 2024-11-16T04:25:27,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:27,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T04:25:27,618 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 6a049fe61dfb22dfb7ca7a9173671d57, server=fda3efb342d8,45455,1731731034087 in 1.1610 sec 2024-11-16T04:25:27,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-16T04:25:27,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6a049fe61dfb22dfb7ca7a9173671d57, UNASSIGN in 1.1650 sec 2024-11-16T04:25:27,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T04:25:27,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-16T04:25:27,621 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731127621"}]},"ts":"1731731127621"} 2024-11-16T04:25:27,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T04:25:27,623 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T04:25:27,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-16T04:25:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T04:25:28,594 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:28,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T04:25:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,602 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,602 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T04:25:28,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:28,606 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/recovered.edits] 2024-11-16T04:25:28,609 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1471cc9cbaa24e7a99bc1ac2398d7125 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/1471cc9cbaa24e7a99bc1ac2398d7125 2024-11-16T04:25:28,610 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4f33f40f58ce401abd7778e603c15081 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/4f33f40f58ce401abd7778e603c15081 2024-11-16T04:25:28,611 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/83a0e08874cd4e8d87af26c42d92b05b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/A/83a0e08874cd4e8d87af26c42d92b05b 2024-11-16T04:25:28,614 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/294e861a72284be0ac4691b6c5bf10d3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/294e861a72284be0ac4691b6c5bf10d3 2024-11-16T04:25:28,615 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6fc2ddf3386646f4bcd35d6a89d9696a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/6fc2ddf3386646f4bcd35d6a89d9696a 2024-11-16T04:25:28,617 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/b61f0f3071154efaaf95ad2be8b2094a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/B/b61f0f3071154efaaf95ad2be8b2094a 2024-11-16T04:25:28,619 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/56dd3effadf94bfcaf6ac05d566107c2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/56dd3effadf94bfcaf6ac05d566107c2 2024-11-16T04:25:28,621 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba1bfc7f72aa429fb089ed307f5839b4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/ba1bfc7f72aa429fb089ed307f5839b4 2024-11-16T04:25:28,623 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f630cb196822475eba2475b962f8e907 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/C/f630cb196822475eba2475b962f8e907 2024-11-16T04:25:28,626 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/recovered.edits/402.seqid to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57/recovered.edits/402.seqid 2024-11-16T04:25:28,627 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/6a049fe61dfb22dfb7ca7a9173671d57 2024-11-16T04:25:28,627 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T04:25:28,629 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,635 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T04:25:28,637 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T04:25:28,638 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,638 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T04:25:28,639 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731731128639"}]},"ts":"9223372036854775807"} 2024-11-16T04:25:28,641 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T04:25:28,641 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6a049fe61dfb22dfb7ca7a9173671d57, NAME => 'TestAcidGuarantees,,1731731101926.6a049fe61dfb22dfb7ca7a9173671d57.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T04:25:28,642 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T04:25:28,642 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731731128642"}]},"ts":"9223372036854775807"} 2024-11-16T04:25:28,645 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T04:25:28,646 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,648 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 52 msec 2024-11-16T04:25:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T04:25:28,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T04:25:28,716 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:28,725 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 238), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=464 (was 409) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8221 (was 8504) 2024-11-16T04:25:28,735 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=464, ProcessCount=11, AvailableMemoryMB=8220 2024-11-16T04:25:28,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:25:28,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:25:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:28,739 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T04:25:28,740 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:28,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-16T04:25:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-16T04:25:28,741 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T04:25:28,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742162_1338 (size=959) 2024-11-16T04:25:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-16T04:25:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-16T04:25:29,149 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:25:29,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742163_1339 (size=53) 2024-11-16T04:25:29,163 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:29,164 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5e2b8a1f3e14054afb97c94e793d2488, disabling compactions & flushes 2024-11-16T04:25:29,164 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,164 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,164 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. after waiting 0 ms 2024-11-16T04:25:29,164 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,164 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,164 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e2b8a1f3e14054afb97c94e793d2488: Waiting for close lock at 1731731129163Disabling compacts and flushes for region at 1731731129163Disabling writes for close at 1731731129164 (+1 ms)Writing region close event to WAL at 1731731129164Closed at 1731731129164 2024-11-16T04:25:29,165 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T04:25:29,166 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731731129165"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731731129165"}]},"ts":"1731731129165"} 2024-11-16T04:25:29,168 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T04:25:29,169 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T04:25:29,169 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731129169"}]},"ts":"1731731129169"} 2024-11-16T04:25:29,171 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T04:25:29,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, ASSIGN}] 2024-11-16T04:25:29,174 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, ASSIGN 2024-11-16T04:25:29,175 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, ASSIGN; state=OFFLINE, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=false 2024-11-16T04:25:29,326 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:29,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, ASSIGN because future has completed 2024-11-16T04:25:29,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-16T04:25:29,483 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,484 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:25:29,484 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,484 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:29,484 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,484 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,486 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,487 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e2b8a1f3e14054afb97c94e793d2488 columnFamilyName A 2024-11-16T04:25:29,487 DEBUG [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:29,488 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(327): Store=5e2b8a1f3e14054afb97c94e793d2488/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:29,488 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,489 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e2b8a1f3e14054afb97c94e793d2488 columnFamilyName B 2024-11-16T04:25:29,489 DEBUG [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:29,490 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(327): Store=5e2b8a1f3e14054afb97c94e793d2488/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:29,490 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,493 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e2b8a1f3e14054afb97c94e793d2488 columnFamilyName C 2024-11-16T04:25:29,493 DEBUG [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:29,494 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(327): Store=5e2b8a1f3e14054afb97c94e793d2488/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:29,494 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,495 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,495 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,497 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,497 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,497 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:25:29,499 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,505 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:25:29,505 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened 5e2b8a1f3e14054afb97c94e793d2488; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67411625, jitterRate=0.00451149046421051}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:25:29,505 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:29,506 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for 5e2b8a1f3e14054afb97c94e793d2488: Running coprocessor pre-open hook at 1731731129484Writing region info on filesystem at 1731731129484Initializing all the Stores at 1731731129485 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731129485Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731129485Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731129485Cleaning up temporary data from old regions at 1731731129497 (+12 ms)Running coprocessor post-open hooks at 1731731129505 (+8 ms)Region opened successfully at 1731731129506 (+1 ms) 2024-11-16T04:25:29,507 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., pid=87, masterSystemTime=1731731129480 2024-11-16T04:25:29,509 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,509 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:29,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:29,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:29,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T04:25:29,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 in 185 msec 2024-11-16T04:25:29,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-16T04:25:29,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, ASSIGN in 344 msec 2024-11-16T04:25:29,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T04:25:29,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731129520"}]},"ts":"1731731129520"} 2024-11-16T04:25:29,523 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T04:25:29,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T04:25:29,526 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 787 msec 2024-11-16T04:25:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-16T04:25:29,874 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:29,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500bbd5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:29,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:29,875 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:29,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@497fed3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:29,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:29,878 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:29,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f648df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:29,879 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:29,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:29,881 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:25:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:25:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4db49071 2024-11-16T04:25:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:25:29,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:25:29,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:25:29,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:25:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:29,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742164_1340 (size=995) 2024-11-16T04:25:30,296 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T04:25:30,297 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T04:25:30,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:25:30,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, REOPEN/MOVE}] 2024-11-16T04:25:30,302 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, REOPEN/MOVE 2024-11-16T04:25:30,303 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, REOPEN/MOVE because future has completed 2024-11-16T04:25:30,305 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:25:30,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:30,306 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45087 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=fda3efb342d8,45455,1731731034087, table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T04:25:30,457 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,457 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:25:30,457 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 5e2b8a1f3e14054afb97c94e793d2488, disabling compactions & flushes 2024-11-16T04:25:30,457 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,457 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,457 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. after waiting 0 ms 2024-11-16T04:25:30,457 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,460 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T04:25:30,461 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,461 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 5e2b8a1f3e14054afb97c94e793d2488: Waiting for close lock at 1731731130457Running coprocessor pre-close hooks at 1731731130457Disabling compacts and flushes for region at 1731731130457Disabling writes for close at 1731731130457Writing region close event to WAL at 1731731130458 (+1 ms)Running coprocessor post-close hooks at 1731731130461 (+3 ms)Closed at 1731731130461 2024-11-16T04:25:30,461 WARN [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: 5e2b8a1f3e14054afb97c94e793d2488 to self. 2024-11-16T04:25:30,463 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=CLOSED 2024-11-16T04:25:30,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:30,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T04:25:30,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 in 160 msec 2024-11-16T04:25:30,467 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, REOPEN/MOVE; state=CLOSED, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=true 2024-11-16T04:25:30,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, REOPEN/MOVE because future has completed 2024-11-16T04:25:30,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:30,774 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,774 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:25:30,774 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,774 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:30,774 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,774 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,775 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,776 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e2b8a1f3e14054afb97c94e793d2488 columnFamilyName A 2024-11-16T04:25:30,777 DEBUG [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:30,777 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(327): Store=5e2b8a1f3e14054afb97c94e793d2488/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:30,778 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,778 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e2b8a1f3e14054afb97c94e793d2488 columnFamilyName B 2024-11-16T04:25:30,778 DEBUG [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:30,779 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(327): Store=5e2b8a1f3e14054afb97c94e793d2488/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:30,779 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,779 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e2b8a1f3e14054afb97c94e793d2488 columnFamilyName C 2024-11-16T04:25:30,779 DEBUG [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:30,780 INFO [StoreOpener-5e2b8a1f3e14054afb97c94e793d2488-1 {}] regionserver.HStore(327): Store=5e2b8a1f3e14054afb97c94e793d2488/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:30,780 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,780 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,781 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,782 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,782 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,782 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:25:30,783 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,784 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened 5e2b8a1f3e14054afb97c94e793d2488; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69692376, jitterRate=0.038497328758239746}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:25:30,784 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,784 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for 5e2b8a1f3e14054afb97c94e793d2488: Running coprocessor pre-open hook at 1731731130774Writing region info on filesystem at 1731731130774Initializing all the Stores at 1731731130775 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731130775Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731130775Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731130775Cleaning up temporary data from old regions at 1731731130782 (+7 ms)Running coprocessor post-open hooks at 1731731130784 (+2 ms)Region opened successfully at 1731731130784 2024-11-16T04:25:30,785 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., pid=92, masterSystemTime=1731731130771 2024-11-16T04:25:30,789 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,789 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,790 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=OPEN, openSeqNum=5, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:30,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-16T04:25:30,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 in 173 msec 2024-11-16T04:25:30,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-16T04:25:30,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, REOPEN/MOVE in 493 msec 2024-11-16T04:25:30,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T04:25:30,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-16T04:25:30,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-16T04:25:30,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T04:25:30,803 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:30,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270218ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,805 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e39ea2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b44e8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a8a339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f0c9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6960d7fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f785f54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@153c9195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4755abf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b285eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,816 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec8001a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,818 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c1947cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d7a2e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,820 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570fed93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,822 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ea97aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ac50c9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb56415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f50c597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f33055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,827 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d952d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7a47ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfb46db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fac7301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,831 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abe5363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e87c032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e92184, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,835 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1ccfe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f344892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:30,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:30,837 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0fd95b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:30,838 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a92a572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:30,840 DEBUG [Thread-1564 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,840 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,840 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,840 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,840 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:30,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,842 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-16T04:25:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:30,844 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:30,844 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,844 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:30,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,848 DEBUG [Thread-1562 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,849 DEBUG [Thread-1568 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,849 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,853 DEBUG [Thread-1562 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,853 DEBUG [Thread-1562 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,854 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,855 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:30,859 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:30,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:25:30,860 DEBUG [Thread-1566 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,864 DEBUG [Thread-1566 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,864 DEBUG [Thread-1566 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,867 DEBUG [Thread-1566 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,868 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:30,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:30,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:25:30,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:30,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731190903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731190909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731190912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:30,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731190915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731190915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:30,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636e3e3c26e73409a858089a1167ffad1_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731130858/Put/seqid=0 2024-11-16T04:25:30,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742165_1341 (size=9714) 2024-11-16T04:25:30,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:30,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:30,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731191024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731191025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731191034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731191034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731191034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:31,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731191233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731191233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731191244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731191244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731191244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,353 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:31,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636e3e3c26e73409a858089a1167ffad1_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636e3e3c26e73409a858089a1167ffad1_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:31,358 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/01c7a1e89e0a4243a8bf7d7cf728c5fd, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:31,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/01c7a1e89e0a4243a8bf7d7cf728c5fd is 175, key is test_row_0/A:col10/1731731130858/Put/seqid=0 2024-11-16T04:25:31,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742166_1342 (size=22361) 2024-11-16T04:25:31,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:31,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731191554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731191554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731191554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731191554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:31,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731191564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:31,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:31,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:31,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,783 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/01c7a1e89e0a4243a8bf7d7cf728c5fd 2024-11-16T04:25:31,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/e806f354298c4b75945b79d9a0e5148a is 50, key is test_row_0/B:col10/1731731130858/Put/seqid=0 2024-11-16T04:25:31,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742167_1343 (size=9657) 2024-11-16T04:25:31,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:31,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731192074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731192074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:32,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731192074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:32,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731192074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:32,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:32,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:32,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:32,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:32,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:32,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:32,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731192085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:32,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:32,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:32,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:32,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/e806f354298c4b75945b79d9a0e5148a 2024-11-16T04:25:32,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/7a33c2d4ce834ad88e27067523bf7678 is 50, key is test_row_0/C:col10/1731731130858/Put/seqid=0 2024-11-16T04:25:32,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742168_1344 (size=9657) 2024-11-16T04:25:32,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/7a33c2d4ce834ad88e27067523bf7678 2024-11-16T04:25:32,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/01c7a1e89e0a4243a8bf7d7cf728c5fd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd 2024-11-16T04:25:32,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd, entries=100, sequenceid=17, filesize=21.8 K 2024-11-16T04:25:32,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/e806f354298c4b75945b79d9a0e5148a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e806f354298c4b75945b79d9a0e5148a 2024-11-16T04:25:32,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e806f354298c4b75945b79d9a0e5148a, entries=100, sequenceid=17, filesize=9.4 K 2024-11-16T04:25:32,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/7a33c2d4ce834ad88e27067523bf7678 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7a33c2d4ce834ad88e27067523bf7678 2024-11-16T04:25:32,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7a33c2d4ce834ad88e27067523bf7678, entries=100, sequenceid=17, filesize=9.4 K 2024-11-16T04:25:32,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5e2b8a1f3e14054afb97c94e793d2488 in 1481ms, sequenceid=17, compaction requested=false 2024-11-16T04:25:32,341 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T04:25:32,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:32,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-16T04:25:32,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:32,390 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:25:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161ab9fe6dcae34a67baaa59b7ace938dd_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731130907/Put/seqid=0 2024-11-16T04:25:32,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742169_1345 (size=12154) 2024-11-16T04:25:32,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:32,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161ab9fe6dcae34a67baaa59b7ace938dd_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ab9fe6dcae34a67baaa59b7ace938dd_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:32,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/b57445b92fb342a7b3aeb68ee5d61e58, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:32,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/b57445b92fb342a7b3aeb68ee5d61e58 is 175, key is test_row_0/A:col10/1731731130907/Put/seqid=0 2024-11-16T04:25:32,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742170_1346 (size=30955) 2024-11-16T04:25:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:33,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:33,134 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:25:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731193108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731193107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731193138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731193138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731193138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,241 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/b57445b92fb342a7b3aeb68ee5d61e58 2024-11-16T04:25:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/2a7583c97f764b519a3ff2b114bf3aa5 is 50, key is test_row_0/B:col10/1731731130907/Put/seqid=0 2024-11-16T04:25:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731193244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731193244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742171_1347 (size=12001) 2024-11-16T04:25:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731193253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731193253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731193253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731193453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731193453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731193463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731193463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731193463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/2a7583c97f764b519a3ff2b114bf3aa5 2024-11-16T04:25:33,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/942ac4b5732b4576a2f3be60f374ae9d is 50, key is test_row_0/C:col10/1731731130907/Put/seqid=0 2024-11-16T04:25:33,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742172_1348 (size=12001) 2024-11-16T04:25:33,664 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/942ac4b5732b4576a2f3be60f374ae9d 2024-11-16T04:25:33,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/b57445b92fb342a7b3aeb68ee5d61e58 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58 2024-11-16T04:25:33,673 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58, entries=150, sequenceid=40, filesize=30.2 K 2024-11-16T04:25:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/2a7583c97f764b519a3ff2b114bf3aa5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/2a7583c97f764b519a3ff2b114bf3aa5 2024-11-16T04:25:33,678 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/2a7583c97f764b519a3ff2b114bf3aa5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T04:25:33,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/942ac4b5732b4576a2f3be60f374ae9d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/942ac4b5732b4576a2f3be60f374ae9d 2024-11-16T04:25:33,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/942ac4b5732b4576a2f3be60f374ae9d, entries=150, sequenceid=40, filesize=11.7 K 2024-11-16T04:25:33,683 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 5e2b8a1f3e14054afb97c94e793d2488 in 1293ms, sequenceid=40, compaction requested=false 2024-11-16T04:25:33,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:33,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:33,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-16T04:25:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-16T04:25:33,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T04:25:33,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8400 sec 2024-11-16T04:25:33,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.8450 sec 2024-11-16T04:25:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:33,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T04:25:33,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166116aaea12eb4a0383d0c0dd25f387f9_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:33,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742173_1349 (size=14594) 2024-11-16T04:25:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731193796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731193797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731193798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731193804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731193805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731193913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731193913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731193913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731193913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:33,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:33,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731193914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:33,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731194123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731194123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731194123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731194124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731194133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,178 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:34,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166116aaea12eb4a0383d0c0dd25f387f9_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166116aaea12eb4a0383d0c0dd25f387f9_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:34,181 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/811a71f020f942fd81f37b91d6503da7, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:34,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/811a71f020f942fd81f37b91d6503da7 is 175, key is test_row_0/A:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:34,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742174_1350 (size=39549) 2024-11-16T04:25:34,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731194433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731194433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731194433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731194433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731194454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,585 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/811a71f020f942fd81f37b91d6503da7 2024-11-16T04:25:34,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/042b88d3f5f14f0e8358b5f243c5f43d is 50, key is test_row_0/B:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742175_1351 (size=12001) 2024-11-16T04:25:34,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/042b88d3f5f14f0e8358b5f243c5f43d 2024-11-16T04:25:34,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/a04fda3a408246bf97ca669f13dbc538 is 50, key is test_row_0/C:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:34,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742176_1352 (size=12001) 2024-11-16T04:25:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731194943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731194943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731194954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731194954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:34,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731194973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:34,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T04:25:35,004 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:35,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:35,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/a04fda3a408246bf97ca669f13dbc538 2024-11-16T04:25:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-16T04:25:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T04:25:35,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:35,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:35,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:35,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/811a71f020f942fd81f37b91d6503da7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7 2024-11-16T04:25:35,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7, entries=200, sequenceid=54, filesize=38.6 K 2024-11-16T04:25:35,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/042b88d3f5f14f0e8358b5f243c5f43d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/042b88d3f5f14f0e8358b5f243c5f43d 2024-11-16T04:25:35,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/042b88d3f5f14f0e8358b5f243c5f43d, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T04:25:35,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/a04fda3a408246bf97ca669f13dbc538 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a04fda3a408246bf97ca669f13dbc538 2024-11-16T04:25:35,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a04fda3a408246bf97ca669f13dbc538, entries=150, sequenceid=54, filesize=11.7 K 2024-11-16T04:25:35,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 5e2b8a1f3e14054afb97c94e793d2488 in 1260ms, sequenceid=54, compaction requested=true 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:35,026 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:35,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T04:25:35,026 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:35,027 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:35,027 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/A is initiating minor compaction (all files) 2024-11-16T04:25:35,027 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/A in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:35,027 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=90.7 K 2024-11-16T04:25:35,027 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:35,027 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7] 2024-11-16T04:25:35,028 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:35,028 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/B is initiating minor compaction (all files) 2024-11-16T04:25:35,028 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/B in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:35,028 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e806f354298c4b75945b79d9a0e5148a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/2a7583c97f764b519a3ff2b114bf3aa5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/042b88d3f5f14f0e8358b5f243c5f43d] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=32.9 K 2024-11-16T04:25:35,028 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c7a1e89e0a4243a8bf7d7cf728c5fd, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731731130858 2024-11-16T04:25:35,028 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting e806f354298c4b75945b79d9a0e5148a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731731130858 2024-11-16T04:25:35,028 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7583c97f764b519a3ff2b114bf3aa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731731130886 2024-11-16T04:25:35,028 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57445b92fb342a7b3aeb68ee5d61e58, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731731130886 2024-11-16T04:25:35,029 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 042b88d3f5f14f0e8358b5f243c5f43d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731731133101 2024-11-16T04:25:35,029 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 811a71f020f942fd81f37b91d6503da7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731731133101 2024-11-16T04:25:35,033 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:35,037 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116faf8822f6320401c90d8cb6a33388fe1_5e2b8a1f3e14054afb97c94e793d2488 store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:35,038 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116faf8822f6320401c90d8cb6a33388fe1_5e2b8a1f3e14054afb97c94e793d2488, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:35,038 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116faf8822f6320401c90d8cb6a33388fe1_5e2b8a1f3e14054afb97c94e793d2488 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:35,041 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:35,041 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/4629a10e52e64eb5a5d68acc933de648 is 50, key is test_row_0/B:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:35,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742177_1353 (size=4469) 2024-11-16T04:25:35,054 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#A#compaction#300 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:35,055 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/6d0e7495bc904314a99c74af2ade845e is 175, key is test_row_0/A:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:35,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742178_1354 (size=12104) 2024-11-16T04:25:35,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742179_1355 (size=31058) 2024-11-16T04:25:35,076 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/6d0e7495bc904314a99c74af2ade845e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/6d0e7495bc904314a99c74af2ade845e 2024-11-16T04:25:35,082 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/A of 5e2b8a1f3e14054afb97c94e793d2488 into 6d0e7495bc904314a99c74af2ade845e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:35,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:35,082 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/A, priority=13, startTime=1731731135026; duration=0sec 2024-11-16T04:25:35,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:35,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:A 2024-11-16T04:25:35,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:35,083 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:35,083 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/C is initiating minor compaction (all files) 2024-11-16T04:25:35,083 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/C in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:35,083 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7a33c2d4ce834ad88e27067523bf7678, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/942ac4b5732b4576a2f3be60f374ae9d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a04fda3a408246bf97ca669f13dbc538] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=32.9 K 2024-11-16T04:25:35,083 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a33c2d4ce834ad88e27067523bf7678, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731731130858 2024-11-16T04:25:35,083 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 942ac4b5732b4576a2f3be60f374ae9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731731130886 2024-11-16T04:25:35,084 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04fda3a408246bf97ca669f13dbc538, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731731133101 2024-11-16T04:25:35,090 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#C#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:35,091 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/80d916fbd7ba4c24840c16568e47f34e is 50, key is test_row_0/C:col10/1731731133765/Put/seqid=0 2024-11-16T04:25:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742180_1356 (size=12104) 2024-11-16T04:25:35,104 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/80d916fbd7ba4c24840c16568e47f34e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/80d916fbd7ba4c24840c16568e47f34e 2024-11-16T04:25:35,108 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/C of 5e2b8a1f3e14054afb97c94e793d2488 into 80d916fbd7ba4c24840c16568e47f34e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:35,108 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:35,108 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/C, priority=13, startTime=1731731135026; duration=0sec 2024-11-16T04:25:35,108 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:35,108 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:C 2024-11-16T04:25:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T04:25:35,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-16T04:25:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:35,159 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T04:25:35,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163738363d9e004301a93e679451f77d1a_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731133794/Put/seqid=0 2024-11-16T04:25:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742181_1357 (size=12154) 2024-11-16T04:25:35,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:35,178 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163738363d9e004301a93e679451f77d1a_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163738363d9e004301a93e679451f77d1a_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:35,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/be1e4d61fa864659be8c7effc3820502, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:35,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/be1e4d61fa864659be8c7effc3820502 is 175, key is test_row_0/A:col10/1731731133794/Put/seqid=0 2024-11-16T04:25:35,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742182_1358 (size=30955) 2024-11-16T04:25:35,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T04:25:35,468 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/4629a10e52e64eb5a5d68acc933de648 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4629a10e52e64eb5a5d68acc933de648 2024-11-16T04:25:35,472 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/B of 5e2b8a1f3e14054afb97c94e793d2488 into 4629a10e52e64eb5a5d68acc933de648(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:35,472 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:35,472 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/B, priority=13, startTime=1731731135026; duration=0sec 2024-11-16T04:25:35,472 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:35,472 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:B 2024-11-16T04:25:35,586 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/be1e4d61fa864659be8c7effc3820502 2024-11-16T04:25:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/808b79d6d1274c84813ec62a1c5852db is 50, key is test_row_0/B:col10/1731731133794/Put/seqid=0 2024-11-16T04:25:35,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742183_1359 (size=12001) 2024-11-16T04:25:35,597 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/808b79d6d1274c84813ec62a1c5852db 2024-11-16T04:25:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/58a3954cbab64cbaa3a5a16c2a8d5bc8 is 50, key is test_row_0/C:col10/1731731133794/Put/seqid=0 2024-11-16T04:25:35,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742184_1360 (size=12001) 2024-11-16T04:25:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T04:25:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:35,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731195979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731195981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:35,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731195982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:35,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731195984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731195987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:35,994 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,006 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/58a3954cbab64cbaa3a5a16c2a8d5bc8 2024-11-16T04:25:36,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/be1e4d61fa864659be8c7effc3820502 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502 2024-11-16T04:25:36,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502, entries=150, sequenceid=78, filesize=30.2 K 2024-11-16T04:25:36,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/808b79d6d1274c84813ec62a1c5852db as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/808b79d6d1274c84813ec62a1c5852db 2024-11-16T04:25:36,016 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/808b79d6d1274c84813ec62a1c5852db, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T04:25:36,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/58a3954cbab64cbaa3a5a16c2a8d5bc8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/58a3954cbab64cbaa3a5a16c2a8d5bc8 2024-11-16T04:25:36,019 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/58a3954cbab64cbaa3a5a16c2a8d5bc8, entries=150, sequenceid=78, filesize=11.7 K 2024-11-16T04:25:36,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5e2b8a1f3e14054afb97c94e793d2488 in 861ms, sequenceid=78, compaction requested=false 2024-11-16T04:25:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-16T04:25:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-16T04:25:36,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-16T04:25:36,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-16T04:25:36,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0200 sec 2024-11-16T04:25:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:36,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:25:36,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164b70040fb7494ceeb0ed4ab04a2fab85_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:36,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742185_1361 (size=14594) 2024-11-16T04:25:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-16T04:25:36,144 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:36,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-16T04:25:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:36,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:36,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:36,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731196158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731196159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731196173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731196174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:36,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731196283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731196283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:36,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:36,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731196293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731196293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:36,507 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:36,510 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164b70040fb7494ceeb0ed4ab04a2fab85_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164b70040fb7494ceeb0ed4ab04a2fab85_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:36,510 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/319dd42f2915455d8d6ce9b07680c1ee, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:36,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731196503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/319dd42f2915455d8d6ce9b07680c1ee is 175, key is test_row_0/A:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731196503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742186_1362 (size=39549) 2024-11-16T04:25:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731196513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731196513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:36,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731196813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731196823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731196833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731196833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:36,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:36,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:36,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:36,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/319dd42f2915455d8d6ce9b07680c1ee 2024-11-16T04:25:36,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/0eea1a01866e47a7b61dc7bda6c600e9 is 50, key is test_row_0/B:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:36,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742187_1363 (size=12001) 2024-11-16T04:25:37,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:37,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/0eea1a01866e47a7b61dc7bda6c600e9 2024-11-16T04:25:37,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/e585d2a7dac841b8b65d91b1f35adaac is 50, key is test_row_0/C:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:37,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742188_1364 (size=12001) 2024-11-16T04:25:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731197333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731197333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:37,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731197343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731197343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:37,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:37,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:37,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/e585d2a7dac841b8b65d91b1f35adaac 2024-11-16T04:25:37,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/319dd42f2915455d8d6ce9b07680c1ee as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee 2024-11-16T04:25:37,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee, entries=200, sequenceid=94, filesize=38.6 K 2024-11-16T04:25:37,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/0eea1a01866e47a7b61dc7bda6c600e9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/0eea1a01866e47a7b61dc7bda6c600e9 2024-11-16T04:25:37,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/0eea1a01866e47a7b61dc7bda6c600e9, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T04:25:37,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/e585d2a7dac841b8b65d91b1f35adaac as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/e585d2a7dac841b8b65d91b1f35adaac 2024-11-16T04:25:37,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/e585d2a7dac841b8b65d91b1f35adaac, entries=150, sequenceid=94, filesize=11.7 K 2024-11-16T04:25:37,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5e2b8a1f3e14054afb97c94e793d2488 in 1655ms, sequenceid=94, compaction requested=true 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:37,753 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:37,753 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:37,754 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:37,754 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/A is initiating minor compaction (all files) 2024-11-16T04:25:37,754 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/A in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,754 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/6d0e7495bc904314a99c74af2ade845e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=99.2 K 2024-11-16T04:25:37,754 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,754 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/6d0e7495bc904314a99c74af2ade845e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee] 2024-11-16T04:25:37,754 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:37,754 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0e7495bc904314a99c74af2ade845e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731731133101 2024-11-16T04:25:37,754 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/B is initiating minor compaction (all files) 2024-11-16T04:25:37,754 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/B in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,754 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4629a10e52e64eb5a5d68acc933de648, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/808b79d6d1274c84813ec62a1c5852db, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/0eea1a01866e47a7b61dc7bda6c600e9] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=35.3 K 2024-11-16T04:25:37,755 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting be1e4d61fa864659be8c7effc3820502, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731731133777 2024-11-16T04:25:37,755 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4629a10e52e64eb5a5d68acc933de648, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731731133101 2024-11-16T04:25:37,755 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319dd42f2915455d8d6ce9b07680c1ee, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731135975 2024-11-16T04:25:37,755 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 808b79d6d1274c84813ec62a1c5852db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731731133777 2024-11-16T04:25:37,755 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eea1a01866e47a7b61dc7bda6c600e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731135975 2024-11-16T04:25:37,760 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:37,761 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:37,762 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/4f33562d11504fe2ab5ebd235af81696 is 50, key is test_row_0/B:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:37,764 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163f4936b67b5a47c59fdc78abb350683c_5e2b8a1f3e14054afb97c94e793d2488 store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:37,765 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163f4936b67b5a47c59fdc78abb350683c_5e2b8a1f3e14054afb97c94e793d2488, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:37,765 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163f4936b67b5a47c59fdc78abb350683c_5e2b8a1f3e14054afb97c94e793d2488 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:37,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742190_1366 (size=4469) 2024-11-16T04:25:37,774 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#A#compaction#310 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:37,775 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/d094a8e10bd64f80ab40c73ada2d2b2b is 175, key is test_row_0/A:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:37,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742189_1365 (size=12207) 2024-11-16T04:25:37,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742191_1367 (size=31161) 2024-11-16T04:25:37,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-16T04:25:37,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:37,824 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:25:37,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163d9d14f024f948c58fe1b8fbbc9b3cb0_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731136173/Put/seqid=0 2024-11-16T04:25:37,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742192_1368 (size=12154) 2024-11-16T04:25:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:38,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731198085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,181 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/4f33562d11504fe2ab5ebd235af81696 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4f33562d11504fe2ab5ebd235af81696 2024-11-16T04:25:38,185 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/B of 5e2b8a1f3e14054afb97c94e793d2488 into 4f33562d11504fe2ab5ebd235af81696(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:38,185 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:38,185 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/B, priority=13, startTime=1731731137753; duration=0sec 2024-11-16T04:25:38,186 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:38,186 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:B 2024-11-16T04:25:38,186 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:38,189 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:38,189 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/C is initiating minor compaction (all files) 2024-11-16T04:25:38,190 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/C in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:38,190 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/80d916fbd7ba4c24840c16568e47f34e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/58a3954cbab64cbaa3a5a16c2a8d5bc8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/e585d2a7dac841b8b65d91b1f35adaac] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=35.3 K 2024-11-16T04:25:38,190 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d916fbd7ba4c24840c16568e47f34e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731731133101 2024-11-16T04:25:38,195 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 58a3954cbab64cbaa3a5a16c2a8d5bc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731731133777 2024-11-16T04:25:38,196 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/d094a8e10bd64f80ab40c73ada2d2b2b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/d094a8e10bd64f80ab40c73ada2d2b2b 2024-11-16T04:25:38,196 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting e585d2a7dac841b8b65d91b1f35adaac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731135975 2024-11-16T04:25:38,200 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/A of 5e2b8a1f3e14054afb97c94e793d2488 into d094a8e10bd64f80ab40c73ada2d2b2b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:38,200 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:38,200 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/A, priority=13, startTime=1731731137753; duration=0sec 2024-11-16T04:25:38,200 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:38,200 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:A 2024-11-16T04:25:38,206 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:38,206 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/117a28b221fc42f5899e96c8be0ed2d7 is 50, key is test_row_0/C:col10/1731731135975/Put/seqid=0 2024-11-16T04:25:38,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742193_1369 (size=12207) 2024-11-16T04:25:38,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731198208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:38,239 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163d9d14f024f948c58fe1b8fbbc9b3cb0_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163d9d14f024f948c58fe1b8fbbc9b3cb0_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/bb77777d3f0d402eaca794eff8905ba4, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/bb77777d3f0d402eaca794eff8905ba4 is 175, key is test_row_0/A:col10/1731731136173/Put/seqid=0 2024-11-16T04:25:38,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742194_1370 (size=30955) 2024-11-16T04:25:38,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/bb77777d3f0d402eaca794eff8905ba4 2024-11-16T04:25:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/65ba650cc0004247b0310b6fc8138b35 is 50, key is test_row_0/B:col10/1731731136173/Put/seqid=0 2024-11-16T04:25:38,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742195_1371 (size=12001) 2024-11-16T04:25:38,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731198353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731198353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,359 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,359 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731198354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,361 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731198363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,369 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731198424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,613 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/117a28b221fc42f5899e96c8be0ed2d7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/117a28b221fc42f5899e96c8be0ed2d7 2024-11-16T04:25:38,617 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/C of 5e2b8a1f3e14054afb97c94e793d2488 into 117a28b221fc42f5899e96c8be0ed2d7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:38,617 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:38,617 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/C, priority=13, startTime=1731731137753; duration=0sec 2024-11-16T04:25:38,617 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:38,618 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:C 2024-11-16T04:25:38,664 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/65ba650cc0004247b0310b6fc8138b35 2024-11-16T04:25:38,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/694b936789c34b7d8796673bf92f5ad2 is 50, key is test_row_0/C:col10/1731731136173/Put/seqid=0 2024-11-16T04:25:38,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742196_1372 (size=12001) 2024-11-16T04:25:38,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/694b936789c34b7d8796673bf92f5ad2 2024-11-16T04:25:38,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/bb77777d3f0d402eaca794eff8905ba4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4 2024-11-16T04:25:38,682 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4, entries=150, sequenceid=115, filesize=30.2 K 2024-11-16T04:25:38,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/65ba650cc0004247b0310b6fc8138b35 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/65ba650cc0004247b0310b6fc8138b35 2024-11-16T04:25:38,688 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/65ba650cc0004247b0310b6fc8138b35, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:25:38,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/694b936789c34b7d8796673bf92f5ad2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/694b936789c34b7d8796673bf92f5ad2 2024-11-16T04:25:38,691 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/694b936789c34b7d8796673bf92f5ad2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:25:38,692 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5e2b8a1f3e14054afb97c94e793d2488 in 868ms, sequenceid=115, compaction requested=false 2024-11-16T04:25:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-16T04:25:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-16T04:25:38,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T04:25:38,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5470 sec 2024-11-16T04:25:38,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.5520 sec 2024-11-16T04:25:38,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:38,736 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:38,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-16T04:25:38,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bbae9837c84a4e6488065ad9399a4e09_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:38,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742197_1373 (size=14694) 2024-11-16T04:25:38,753 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:38,756 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bbae9837c84a4e6488065ad9399a4e09_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bbae9837c84a4e6488065ad9399a4e09_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:38,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/e032285082ef44ffaded085010dce59e, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:38,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/e032285082ef44ffaded085010dce59e is 175, key is test_row_0/A:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:38,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742198_1374 (size=39649) 2024-11-16T04:25:38,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731198845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:38,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731198964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:39,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/e032285082ef44ffaded085010dce59e 2024-11-16T04:25:39,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/7a3c9468cacd4d1b9cb132326bf20800 is 50, key is test_row_0/B:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:39,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742199_1375 (size=12101) 2024-11-16T04:25:39,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731199173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:39,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731199484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:39,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/7a3c9468cacd4d1b9cb132326bf20800 2024-11-16T04:25:39,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/f024236593dc4f5f8ae2c9fc75ea80ce is 50, key is test_row_0/C:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:39,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742200_1376 (size=12101) 2024-11-16T04:25:39,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731199993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:39,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/f024236593dc4f5f8ae2c9fc75ea80ce 2024-11-16T04:25:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/e032285082ef44ffaded085010dce59e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e 2024-11-16T04:25:40,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e, entries=200, sequenceid=134, filesize=38.7 K 2024-11-16T04:25:40,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/7a3c9468cacd4d1b9cb132326bf20800 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7a3c9468cacd4d1b9cb132326bf20800 2024-11-16T04:25:40,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7a3c9468cacd4d1b9cb132326bf20800, entries=150, sequenceid=134, filesize=11.8 K 2024-11-16T04:25:40,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/f024236593dc4f5f8ae2c9fc75ea80ce as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/f024236593dc4f5f8ae2c9fc75ea80ce 2024-11-16T04:25:40,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/f024236593dc4f5f8ae2c9fc75ea80ce, entries=150, sequenceid=134, filesize=11.8 K 2024-11-16T04:25:40,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5e2b8a1f3e14054afb97c94e793d2488 in 1276ms, sequenceid=134, compaction requested=true 2024-11-16T04:25:40,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:40,013 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:40,013 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:40,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:40,013 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:40,013 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:40,013 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/B is initiating minor compaction (all files) 2024-11-16T04:25:40,013 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/A is initiating minor compaction (all files) 2024-11-16T04:25:40,013 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/B in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,013 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/A in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,013 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4f33562d11504fe2ab5ebd235af81696, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/65ba650cc0004247b0310b6fc8138b35, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7a3c9468cacd4d1b9cb132326bf20800] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=35.5 K 2024-11-16T04:25:40,013 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/d094a8e10bd64f80ab40c73ada2d2b2b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=99.4 K 2024-11-16T04:25:40,013 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/d094a8e10bd64f80ab40c73ada2d2b2b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e] 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f33562d11504fe2ab5ebd235af81696, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731135975 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d094a8e10bd64f80ab40c73ada2d2b2b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731135975 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ba650cc0004247b0310b6fc8138b35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731136128 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb77777d3f0d402eaca794eff8905ba4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731136128 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3c9468cacd4d1b9cb132326bf20800, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731731138051 2024-11-16T04:25:40,014 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e032285082ef44ffaded085010dce59e, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731731138051 2024-11-16T04:25:40,018 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:40,021 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:40,021 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/7f8a3598c2ad4667b21b0f6871d4d005 is 50, key is test_row_0/B:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:40,025 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411169f1cb03809584ecea484ede46cc1ab1d_5e2b8a1f3e14054afb97c94e793d2488 store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:40,027 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411169f1cb03809584ecea484ede46cc1ab1d_5e2b8a1f3e14054afb97c94e793d2488, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:40,027 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169f1cb03809584ecea484ede46cc1ab1d_5e2b8a1f3e14054afb97c94e793d2488 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:40,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742201_1377 (size=12409) 2024-11-16T04:25:40,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742202_1378 (size=4469) 2024-11-16T04:25:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-16T04:25:40,303 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:40,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-16T04:25:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:40,305 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:40,306 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:40,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:40,368 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:40,368 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-16T04:25:40,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d9d576e7e95541188ed1efbad6e42861_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731140367/Put/seqid=0 2024-11-16T04:25:40,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742203_1379 (size=14794) 2024-11-16T04:25:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731200390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731200393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731200398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731200399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:40,434 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/7f8a3598c2ad4667b21b0f6871d4d005 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7f8a3598c2ad4667b21b0f6871d4d005 2024-11-16T04:25:40,434 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:40,435 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/bcc3baa055e3497782b8559e3d387049 is 175, key is test_row_0/A:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:40,438 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/B of 5e2b8a1f3e14054afb97c94e793d2488 into 7f8a3598c2ad4667b21b0f6871d4d005(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:40,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:40,438 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/B, priority=13, startTime=1731731140013; duration=0sec 2024-11-16T04:25:40,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:40,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:B 2024-11-16T04:25:40,438 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:25:40,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742204_1380 (size=31363) 2024-11-16T04:25:40,439 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:25:40,439 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/C is initiating minor compaction (all files) 2024-11-16T04:25:40,439 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/C in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,439 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/117a28b221fc42f5899e96c8be0ed2d7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/694b936789c34b7d8796673bf92f5ad2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/f024236593dc4f5f8ae2c9fc75ea80ce] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=35.5 K 2024-11-16T04:25:40,439 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 117a28b221fc42f5899e96c8be0ed2d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731731135975 2024-11-16T04:25:40,439 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 694b936789c34b7d8796673bf92f5ad2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731136128 2024-11-16T04:25:40,440 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f024236593dc4f5f8ae2c9fc75ea80ce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731731138051 2024-11-16T04:25:40,446 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:40,446 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/9783f47ea76b4ceebbc6839bdd7e1a23 is 50, key is test_row_0/C:col10/1731731138051/Put/seqid=0 2024-11-16T04:25:40,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742205_1381 (size=12409) 2024-11-16T04:25:40,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:40,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731200503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731200503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731200513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731200513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:40,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:40,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731200713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731200713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731200733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731200733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:40,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:40,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,777 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:40,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d9d576e7e95541188ed1efbad6e42861_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d9d576e7e95541188ed1efbad6e42861_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:40,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/f0eb1cf51eff458fab79cee114c45f82, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:40,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/f0eb1cf51eff458fab79cee114c45f82 is 175, key is test_row_0/A:col10/1731731140367/Put/seqid=0 2024-11-16T04:25:40,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742206_1382 (size=39749) 2024-11-16T04:25:40,841 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/bcc3baa055e3497782b8559e3d387049 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bcc3baa055e3497782b8559e3d387049 2024-11-16T04:25:40,845 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/A of 5e2b8a1f3e14054afb97c94e793d2488 into bcc3baa055e3497782b8559e3d387049(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:40,845 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:40,845 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/A, priority=13, startTime=1731731140012; duration=0sec 2024-11-16T04:25:40,845 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:40,845 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:A 2024-11-16T04:25:40,853 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/9783f47ea76b4ceebbc6839bdd7e1a23 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/9783f47ea76b4ceebbc6839bdd7e1a23 2024-11-16T04:25:40,857 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/C of 5e2b8a1f3e14054afb97c94e793d2488 into 9783f47ea76b4ceebbc6839bdd7e1a23(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:40,857 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:40,857 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/C, priority=13, startTime=1731731140013; duration=0sec 2024-11-16T04:25:40,857 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:40,857 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:C 2024-11-16T04:25:40,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:40,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:40,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:41,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731201014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,020 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731201023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731201023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731201043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731201043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/f0eb1cf51eff458fab79cee114c45f82 2024-11-16T04:25:41,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/e2f7853faef74f0abdede1a4fb32ca11 is 50, key is test_row_0/B:col10/1731731140367/Put/seqid=0 2024-11-16T04:25:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742207_1383 (size=12151) 2024-11-16T04:25:41,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:41,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731201533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731201533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731201553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731201563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:41,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:41,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/e2f7853faef74f0abdede1a4fb32ca11 2024-11-16T04:25:41,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/3e20fb9ac4414966ad7b9461b94239ed is 50, key is test_row_0/C:col10/1731731140367/Put/seqid=0 2024-11-16T04:25:41,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742208_1384 (size=12151) 2024-11-16T04:25:41,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:41,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:42,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/3e20fb9ac4414966ad7b9461b94239ed 2024-11-16T04:25:42,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/f0eb1cf51eff458fab79cee114c45f82 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82 2024-11-16T04:25:42,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82, entries=200, sequenceid=155, filesize=38.8 K 2024-11-16T04:25:42,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/e2f7853faef74f0abdede1a4fb32ca11 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e2f7853faef74f0abdede1a4fb32ca11 2024-11-16T04:25:42,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e2f7853faef74f0abdede1a4fb32ca11, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T04:25:42,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/3e20fb9ac4414966ad7b9461b94239ed as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/3e20fb9ac4414966ad7b9461b94239ed 2024-11-16T04:25:42,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/3e20fb9ac4414966ad7b9461b94239ed, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T04:25:42,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5e2b8a1f3e14054afb97c94e793d2488 in 1651ms, sequenceid=155, compaction requested=false 2024-11-16T04:25:42,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:42,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-16T04:25:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:42,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:25:42,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163b90f77c416b412196994db32f777687_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731140398/Put/seqid=0 2024-11-16T04:25:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742209_1385 (size=12304) 2024-11-16T04:25:42,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:42,158 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163b90f77c416b412196994db32f777687_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163b90f77c416b412196994db32f777687_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:42,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/c1b1ebf30bcf411ab27f2c830d910140, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:42,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/c1b1ebf30bcf411ab27f2c830d910140 is 175, key is test_row_0/A:col10/1731731140398/Put/seqid=0 2024-11-16T04:25:42,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742210_1386 (size=31105) 2024-11-16T04:25:42,168 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/c1b1ebf30bcf411ab27f2c830d910140 2024-11-16T04:25:42,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/5988f8ad8a1f4fa09d0b549fc22b4c0a is 50, key is test_row_0/B:col10/1731731140398/Put/seqid=0 2024-11-16T04:25:42,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742211_1387 (size=12151) 2024-11-16T04:25:42,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/5988f8ad8a1f4fa09d0b549fc22b4c0a 2024-11-16T04:25:42,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/0101de8b7c8441dcb46b69915a7241b2 is 50, key is test_row_0/C:col10/1731731140398/Put/seqid=0 2024-11-16T04:25:42,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742212_1388 (size=12151) 2024-11-16T04:25:42,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:42,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731202592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731202592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,602 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/0101de8b7c8441dcb46b69915a7241b2 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731202593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731202597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/c1b1ebf30bcf411ab27f2c830d910140 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140 2024-11-16T04:25:42,608 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140, entries=150, sequenceid=173, filesize=30.4 K 2024-11-16T04:25:42,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/5988f8ad8a1f4fa09d0b549fc22b4c0a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/5988f8ad8a1f4fa09d0b549fc22b4c0a 2024-11-16T04:25:42,612 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/5988f8ad8a1f4fa09d0b549fc22b4c0a, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T04:25:42,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/0101de8b7c8441dcb46b69915a7241b2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/0101de8b7c8441dcb46b69915a7241b2 2024-11-16T04:25:42,616 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/0101de8b7c8441dcb46b69915a7241b2, entries=150, sequenceid=173, filesize=11.9 K 2024-11-16T04:25:42,617 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5e2b8a1f3e14054afb97c94e793d2488 in 480ms, sequenceid=173, compaction requested=true 2024-11-16T04:25:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:42,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-16T04:25:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-16T04:25:42,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-16T04:25:42,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3120 sec 2024-11-16T04:25:42,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3170 sec 2024-11-16T04:25:42,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:42,746 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:42,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:25:42,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162f1483518878453fb016dc97cb492a04_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:42,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742213_1389 (size=14794) 2024-11-16T04:25:42,761 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:42,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162f1483518878453fb016dc97cb492a04_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162f1483518878453fb016dc97cb492a04_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:42,768 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/0a226a0fc2fe4fccb1cded114f68c793, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:42,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/0a226a0fc2fe4fccb1cded114f68c793 is 175, key is test_row_0/A:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:42,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742214_1390 (size=39749) 2024-11-16T04:25:42,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731202768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731202777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731202779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731202779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731202883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731202903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731202903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731202904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:42,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731203034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,038 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731203093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731203113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731203113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731203123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/0a226a0fc2fe4fccb1cded114f68c793 2024-11-16T04:25:43,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/fd2816ea1297457b9a07c4649e4426b5 is 50, key is test_row_0/B:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:43,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742215_1391 (size=12151) 2024-11-16T04:25:43,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731203404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731203423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731203423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731203443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/fd2816ea1297457b9a07c4649e4426b5 2024-11-16T04:25:43,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/12f38f164fcb48e4afc31937fdc54693 is 50, key is test_row_0/C:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:43,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742216_1392 (size=12151) 2024-11-16T04:25:43,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731203923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731203943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731203943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:43,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731203964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:43,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:43,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:43,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/12f38f164fcb48e4afc31937fdc54693 2024-11-16T04:25:44,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/0a226a0fc2fe4fccb1cded114f68c793 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793 2024-11-16T04:25:44,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793, entries=200, sequenceid=198, filesize=38.8 K 2024-11-16T04:25:44,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/fd2816ea1297457b9a07c4649e4426b5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/fd2816ea1297457b9a07c4649e4426b5 2024-11-16T04:25:44,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/fd2816ea1297457b9a07c4649e4426b5, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:25:44,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/12f38f164fcb48e4afc31937fdc54693 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/12f38f164fcb48e4afc31937fdc54693 2024-11-16T04:25:44,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/12f38f164fcb48e4afc31937fdc54693, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:25:44,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 5e2b8a1f3e14054afb97c94e793d2488 in 1267ms, sequenceid=198, compaction requested=true 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:44,014 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:44,014 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:25:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141966 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/A is initiating minor compaction (all files) 2024-11-16T04:25:44,015 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/A in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:44,015 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bcc3baa055e3497782b8559e3d387049, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=138.6 K 2024-11-16T04:25:44,015 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bcc3baa055e3497782b8559e3d387049, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793] 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/B is initiating minor compaction (all files) 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcc3baa055e3497782b8559e3d387049, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731731138051 2024-11-16T04:25:44,015 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/B in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:44,015 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7f8a3598c2ad4667b21b0f6871d4d005, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e2f7853faef74f0abdede1a4fb32ca11, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/5988f8ad8a1f4fa09d0b549fc22b4c0a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/fd2816ea1297457b9a07c4649e4426b5] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=47.7 K 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0eb1cf51eff458fab79cee114c45f82, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731138829 2024-11-16T04:25:44,015 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8a3598c2ad4667b21b0f6871d4d005, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731731138051 2024-11-16T04:25:44,016 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b1ebf30bcf411ab27f2c830d910140, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731731140392 2024-11-16T04:25:44,016 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f7853faef74f0abdede1a4fb32ca11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731138829 2024-11-16T04:25:44,016 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a226a0fc2fe4fccb1cded114f68c793, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731142595 2024-11-16T04:25:44,016 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5988f8ad8a1f4fa09d0b549fc22b4c0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731731140392 2024-11-16T04:25:44,016 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2816ea1297457b9a07c4649e4426b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731142595 2024-11-16T04:25:44,021 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:44,033 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164bc57502315c4b519cced9b22ec3a1d0_5e2b8a1f3e14054afb97c94e793d2488 store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:44,033 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:44,034 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/6513561447d943118bbe9175d74debea is 50, key is test_row_0/B:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:44,035 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164bc57502315c4b519cced9b22ec3a1d0_5e2b8a1f3e14054afb97c94e793d2488, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:44,035 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164bc57502315c4b519cced9b22ec3a1d0_5e2b8a1f3e14054afb97c94e793d2488 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:44,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742217_1393 (size=12595) 2024-11-16T04:25:44,043 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/6513561447d943118bbe9175d74debea as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/6513561447d943118bbe9175d74debea 2024-11-16T04:25:44,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742218_1394 (size=4469) 2024-11-16T04:25:44,046 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#A#compaction#330 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:44,047 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/ed0aee43f4b84729b0917895f0638242 is 175, key is test_row_0/A:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:44,048 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/B of 5e2b8a1f3e14054afb97c94e793d2488 into 6513561447d943118bbe9175d74debea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:44,048 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:44,048 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/B, priority=12, startTime=1731731144014; duration=0sec 2024-11-16T04:25:44,048 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:44,048 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:B 2024-11-16T04:25:44,048 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:25:44,050 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:25:44,050 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/C is initiating minor compaction (all files) 2024-11-16T04:25:44,050 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/C in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:44,050 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/9783f47ea76b4ceebbc6839bdd7e1a23, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/3e20fb9ac4414966ad7b9461b94239ed, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/0101de8b7c8441dcb46b69915a7241b2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/12f38f164fcb48e4afc31937fdc54693] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=47.7 K 2024-11-16T04:25:44,050 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9783f47ea76b4ceebbc6839bdd7e1a23, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731731138051 2024-11-16T04:25:44,051 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e20fb9ac4414966ad7b9461b94239ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731138829 2024-11-16T04:25:44,051 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0101de8b7c8441dcb46b69915a7241b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731731140392 2024-11-16T04:25:44,051 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f38f164fcb48e4afc31937fdc54693, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731142595 2024-11-16T04:25:44,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742219_1395 (size=31549) 2024-11-16T04:25:44,058 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/ed0aee43f4b84729b0917895f0638242 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/ed0aee43f4b84729b0917895f0638242 2024-11-16T04:25:44,062 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/A of 5e2b8a1f3e14054afb97c94e793d2488 into ed0aee43f4b84729b0917895f0638242(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:44,062 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:44,062 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/A, priority=12, startTime=1731731144014; duration=0sec 2024-11-16T04:25:44,062 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:44,062 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:A 2024-11-16T04:25:44,062 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:44,063 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/19ec63690ee94186bb6c24d4af0078fb is 50, key is test_row_0/C:col10/1731731142595/Put/seqid=0 2024-11-16T04:25:44,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742220_1396 (size=12595) 2024-11-16T04:25:44,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-16T04:25:44,464 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:44,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T04:25:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T04:25:44,466 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:44,466 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:44,466 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:44,472 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/19ec63690ee94186bb6c24d4af0078fb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/19ec63690ee94186bb6c24d4af0078fb 2024-11-16T04:25:44,475 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/C of 5e2b8a1f3e14054afb97c94e793d2488 into 19ec63690ee94186bb6c24d4af0078fb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:44,475 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:44,476 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/C, priority=12, startTime=1731731144014; duration=0sec 2024-11-16T04:25:44,476 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:44,476 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:C 2024-11-16T04:25:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T04:25:44,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T04:25:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:44,618 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:25:44,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eced906c2c854a89be5d3b344b21e479_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731142775/Put/seqid=0 2024-11-16T04:25:44,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742221_1397 (size=12304) 2024-11-16T04:25:44,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T04:25:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:44,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:45,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731204994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731204995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731204996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731205001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:45,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eced906c2c854a89be5d3b344b21e479_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eced906c2c854a89be5d3b344b21e479_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:45,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/cfb2e48238c84b3097489d666f0fa55a, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:45,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/cfb2e48238c84b3097489d666f0fa55a is 175, key is test_row_0/A:col10/1731731142775/Put/seqid=0 2024-11-16T04:25:45,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742222_1398 (size=31105) 2024-11-16T04:25:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T04:25:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731205104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731205104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731205113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731205113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731205313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731205314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731205323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731205323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,435 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/cfb2e48238c84b3097489d666f0fa55a 2024-11-16T04:25:45,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/55e87edc0d9142b1bbc57f4ba132581a is 50, key is test_row_0/B:col10/1731731142775/Put/seqid=0 2024-11-16T04:25:45,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742223_1399 (size=12151) 2024-11-16T04:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T04:25:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731205623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731205623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731205634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731205634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:45,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/55e87edc0d9142b1bbc57f4ba132581a 2024-11-16T04:25:45,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/7baf2c13a9474bcf8b90a04f55f85293 is 50, key is test_row_0/C:col10/1731731142775/Put/seqid=0 2024-11-16T04:25:45,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742224_1400 (size=12151) 2024-11-16T04:25:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731206133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731206144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731206144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731206144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:46,253 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/7baf2c13a9474bcf8b90a04f55f85293 2024-11-16T04:25:46,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/cfb2e48238c84b3097489d666f0fa55a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a 2024-11-16T04:25:46,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a, entries=150, sequenceid=213, filesize=30.4 K 2024-11-16T04:25:46,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/55e87edc0d9142b1bbc57f4ba132581a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/55e87edc0d9142b1bbc57f4ba132581a 2024-11-16T04:25:46,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/55e87edc0d9142b1bbc57f4ba132581a, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T04:25:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/7baf2c13a9474bcf8b90a04f55f85293 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7baf2c13a9474bcf8b90a04f55f85293 2024-11-16T04:25:46,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7baf2c13a9474bcf8b90a04f55f85293, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T04:25:46,268 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5e2b8a1f3e14054afb97c94e793d2488 in 1650ms, sequenceid=213, compaction requested=false 2024-11-16T04:25:46,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:46,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:46,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T04:25:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T04:25:46,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T04:25:46,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-11-16T04:25:46,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8070 sec 2024-11-16T04:25:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T04:25:46,613 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:46,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T04:25:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T04:25:46,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:46,616 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:46,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T04:25:46,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T04:25:46,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:46,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:25:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161ae8c67ba0f64eafa5011b9db060d5f0_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731144999/Put/seqid=0 2024-11-16T04:25:46,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742225_1401 (size=12304) 2024-11-16T04:25:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T04:25:47,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:47,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:47,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731207105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731207143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,147 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731207153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,159 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731207164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,168 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731207164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,170 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:47,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161ae8c67ba0f64eafa5011b9db060d5f0_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ae8c67ba0f64eafa5011b9db060d5f0_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/1f7668f9695a4e119506d6bd3d6d7c30, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/1f7668f9695a4e119506d6bd3d6d7c30 is 175, key is test_row_0/A:col10/1731731144999/Put/seqid=0 2024-11-16T04:25:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742226_1402 (size=31105) 2024-11-16T04:25:47,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731207213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T04:25:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731207423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,586 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/1f7668f9695a4e119506d6bd3d6d7c30 2024-11-16T04:25:47,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/468d98dfcb8d4464bba8d95fb451e72e is 50, key is test_row_0/B:col10/1731731144999/Put/seqid=0 2024-11-16T04:25:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742227_1403 (size=12151) 2024-11-16T04:25:47,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731207733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T04:25:47,996 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/468d98dfcb8d4464bba8d95fb451e72e 2024-11-16T04:25:48,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/ae39aec1940541fe9303fdd48819690c is 50, key is test_row_0/C:col10/1731731144999/Put/seqid=0 2024-11-16T04:25:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742228_1404 (size=12151) 2024-11-16T04:25:48,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731208254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:48,449 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/ae39aec1940541fe9303fdd48819690c 2024-11-16T04:25:48,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/1f7668f9695a4e119506d6bd3d6d7c30 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30 2024-11-16T04:25:48,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30, entries=150, sequenceid=237, filesize=30.4 K 2024-11-16T04:25:48,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/468d98dfcb8d4464bba8d95fb451e72e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/468d98dfcb8d4464bba8d95fb451e72e 2024-11-16T04:25:48,458 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/468d98dfcb8d4464bba8d95fb451e72e, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T04:25:48,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/ae39aec1940541fe9303fdd48819690c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/ae39aec1940541fe9303fdd48819690c 2024-11-16T04:25:48,462 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/ae39aec1940541fe9303fdd48819690c, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T04:25:48,464 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5e2b8a1f3e14054afb97c94e793d2488 in 1695ms, sequenceid=237, compaction requested=true 2024-11-16T04:25:48,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:48,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:48,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T04:25:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T04:25:48,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T04:25:48,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-11-16T04:25:48,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8530 sec 2024-11-16T04:25:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T04:25:48,764 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:48,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T04:25:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T04:25:48,766 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:48,767 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:48,767 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T04:25:48,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T04:25:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:48,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:25:48,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e78952fec5df4e0f86a7e8665325c315_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731147104/Put/seqid=0 2024-11-16T04:25:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742229_1405 (size=12304) 2024-11-16T04:25:48,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:48,937 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e78952fec5df4e0f86a7e8665325c315_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e78952fec5df4e0f86a7e8665325c315_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/99d33bc992ce487b88c6256cb6edc598, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:48,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/99d33bc992ce487b88c6256cb6edc598 is 175, key is test_row_0/A:col10/1731731147104/Put/seqid=0 2024-11-16T04:25:48,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742230_1406 (size=31105) 2024-11-16T04:25:49,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T04:25:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:49,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. as already flushing 2024-11-16T04:25:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731209222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731209223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731209227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731209228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731209273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,274 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/99d33bc992ce487b88c6256cb6edc598 2024-11-16T04:25:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731209344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731209344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731209344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731209344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/a6fb2da1ae48447fa76abc7446d34a21 is 50, key is test_row_0/B:col10/1731731147104/Put/seqid=0 2024-11-16T04:25:49,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742231_1407 (size=12151) 2024-11-16T04:25:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T04:25:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731209553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731209553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731209553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731209553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,752 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/a6fb2da1ae48447fa76abc7446d34a21 2024-11-16T04:25:49,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/874d68a0ba97439baeecae91e23b71c8 is 50, key is test_row_0/C:col10/1731731147104/Put/seqid=0 2024-11-16T04:25:49,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742232_1408 (size=12151) 2024-11-16T04:25:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731209863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731209863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731209863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731209864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:49,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T04:25:50,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/874d68a0ba97439baeecae91e23b71c8 2024-11-16T04:25:50,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/99d33bc992ce487b88c6256cb6edc598 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598 2024-11-16T04:25:50,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598, entries=150, sequenceid=249, filesize=30.4 K 2024-11-16T04:25:50,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/a6fb2da1ae48447fa76abc7446d34a21 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/a6fb2da1ae48447fa76abc7446d34a21 2024-11-16T04:25:50,169 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/a6fb2da1ae48447fa76abc7446d34a21, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T04:25:50,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/874d68a0ba97439baeecae91e23b71c8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/874d68a0ba97439baeecae91e23b71c8 2024-11-16T04:25:50,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/874d68a0ba97439baeecae91e23b71c8, entries=150, sequenceid=249, filesize=11.9 K 2024-11-16T04:25:50,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5e2b8a1f3e14054afb97c94e793d2488 in 1254ms, sequenceid=249, compaction requested=true 2024-11-16T04:25:50,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:50,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:50,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T04:25:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T04:25:50,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T04:25:50,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-16T04:25:50,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.4130 sec 2024-11-16T04:25:50,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:50,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:50,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:25:50,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161bc5088c75c64258a100726b5234d5ce_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:50,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742233_1409 (size=17534) 2024-11-16T04:25:50,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731210391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731210392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731210393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731210394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731210504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731210504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731210513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731210513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731210713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731210714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731210733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731210733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:50,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:50,788 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:50,790 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161bc5088c75c64258a100726b5234d5ce_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161bc5088c75c64258a100726b5234d5ce_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:50,791 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/9a98dbb859a64bcdbb9f6bc01ed442c3, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:50,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/9a98dbb859a64bcdbb9f6bc01ed442c3 is 175, key is test_row_0/A:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:50,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742234_1410 (size=48639) 2024-11-16T04:25:50,869 INFO [Thread-1564 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1564. 2024-11-16T04:25:50,869 DEBUG [Thread-1564 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:50,869 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,869 DEBUG [Thread-1564 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:50,874 INFO [Thread-1562 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1562. 2024-11-16T04:25:50,874 DEBUG [Thread-1562 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:50,874 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,874 DEBUG [Thread-1562 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:50,878 INFO [Thread-1568 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1568. 2024-11-16T04:25:50,878 DEBUG [Thread-1568 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:50,878 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,878 DEBUG [Thread-1568 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:50,879 INFO [Thread-1566 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1566. 2024-11-16T04:25:50,879 DEBUG [Thread-1566 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:50,879 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,879 DEBUG [Thread-1566 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:50,883 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-16T04:25:50,883 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:50,883 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,883 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:50,884 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T04:25:50,914 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731211024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731211024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731211044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731211044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,196 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/9a98dbb859a64bcdbb9f6bc01ed442c3 2024-11-16T04:25:51,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/814b28a5966a47768bd855651597229c is 50, key is test_row_0/B:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:51,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742235_1411 (size=12301) 2024-11-16T04:25:51,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54216 deadline: 1731731211294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,295 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 5e2b8a1f3e14054afb97c94e793d2488 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:25:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54260 deadline: 1731731211534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54300 deadline: 1731731211534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54248 deadline: 1731731211553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54272 deadline: 1731731211553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:25:51,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/814b28a5966a47768bd855651597229c 2024-11-16T04:25:51,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/460dbd12518f44e793dc1a280a384a23 is 50, key is test_row_0/C:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:51,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742236_1412 (size=12301) 2024-11-16T04:25:52,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/460dbd12518f44e793dc1a280a384a23 2024-11-16T04:25:52,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/9a98dbb859a64bcdbb9f6bc01ed442c3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3 2024-11-16T04:25:52,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3, entries=250, sequenceid=275, filesize=47.5 K 2024-11-16T04:25:52,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/814b28a5966a47768bd855651597229c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/814b28a5966a47768bd855651597229c 2024-11-16T04:25:52,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/814b28a5966a47768bd855651597229c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T04:25:52,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/460dbd12518f44e793dc1a280a384a23 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/460dbd12518f44e793dc1a280a384a23 2024-11-16T04:25:52,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/460dbd12518f44e793dc1a280a384a23, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T04:25:52,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5e2b8a1f3e14054afb97c94e793d2488 in 1673ms, sequenceid=275, compaction requested=true 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:25:52,051 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e2b8a1f3e14054afb97c94e793d2488:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:25:52,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:52,051 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T04:25:52,052 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173503 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T04:25:52,052 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T04:25:52,052 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/A is initiating minor compaction (all files) 2024-11-16T04:25:52,052 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/B is initiating minor compaction (all files) 2024-11-16T04:25:52,053 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/A in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:52,053 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/B in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:52,053 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/ed0aee43f4b84729b0917895f0638242, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=169.4 K 2024-11-16T04:25:52,053 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/6513561447d943118bbe9175d74debea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/55e87edc0d9142b1bbc57f4ba132581a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/468d98dfcb8d4464bba8d95fb451e72e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/a6fb2da1ae48447fa76abc7446d34a21, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/814b28a5966a47768bd855651597229c] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=59.9 K 2024-11-16T04:25:52,053 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:52,053 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/ed0aee43f4b84729b0917895f0638242, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3] 2024-11-16T04:25:52,053 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 6513561447d943118bbe9175d74debea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731142595 2024-11-16T04:25:52,053 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0aee43f4b84729b0917895f0638242, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731142595 2024-11-16T04:25:52,053 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e87edc0d9142b1bbc57f4ba132581a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731731142775 2024-11-16T04:25:52,053 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb2e48238c84b3097489d666f0fa55a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731731142775 2024-11-16T04:25:52,054 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 468d98dfcb8d4464bba8d95fb451e72e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731731144994 2024-11-16T04:25:52,054 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f7668f9695a4e119506d6bd3d6d7c30, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731731144994 2024-11-16T04:25:52,054 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a6fb2da1ae48447fa76abc7446d34a21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731147066 2024-11-16T04:25:52,054 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d33bc992ce487b88c6256cb6edc598, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731147066 2024-11-16T04:25:52,054 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 814b28a5966a47768bd855651597229c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731731149226 2024-11-16T04:25:52,054 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a98dbb859a64bcdbb9f6bc01ed442c3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731731149210 2024-11-16T04:25:52,061 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:52,062 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116400ee4d48e4d4a10b7842d91c8eb7bad_5e2b8a1f3e14054afb97c94e793d2488 store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:52,062 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:52,063 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/f03c011fedcf4c93a076a9fd9c6b2009 is 50, key is test_row_0/B:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:52,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742237_1413 (size=12915) 2024-11-16T04:25:52,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116400ee4d48e4d4a10b7842d91c8eb7bad_5e2b8a1f3e14054afb97c94e793d2488, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:52,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116400ee4d48e4d4a10b7842d91c8eb7bad_5e2b8a1f3e14054afb97c94e793d2488 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:52,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742238_1414 (size=4469) 2024-11-16T04:25:52,337 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T04:25:52,471 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/f03c011fedcf4c93a076a9fd9c6b2009 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f03c011fedcf4c93a076a9fd9c6b2009 2024-11-16T04:25:52,472 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:52,473 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/79894e975ee54ef2850fc40004db4da2 is 175, key is test_row_0/A:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:52,475 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/B of 5e2b8a1f3e14054afb97c94e793d2488 into f03c011fedcf4c93a076a9fd9c6b2009(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:52,475 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:52,475 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/B, priority=11, startTime=1731731152051; duration=0sec 2024-11-16T04:25:52,475 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:25:52,475 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:B 2024-11-16T04:25:52,475 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T04:25:52,477 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61349 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T04:25:52,477 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 5e2b8a1f3e14054afb97c94e793d2488/C is initiating minor compaction (all files) 2024-11-16T04:25:52,477 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e2b8a1f3e14054afb97c94e793d2488/C in TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:52,477 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/19ec63690ee94186bb6c24d4af0078fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7baf2c13a9474bcf8b90a04f55f85293, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/ae39aec1940541fe9303fdd48819690c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/874d68a0ba97439baeecae91e23b71c8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/460dbd12518f44e793dc1a280a384a23] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp, totalSize=59.9 K 2024-11-16T04:25:52,477 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 19ec63690ee94186bb6c24d4af0078fb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731142595 2024-11-16T04:25:52,477 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7baf2c13a9474bcf8b90a04f55f85293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731731142775 2024-11-16T04:25:52,478 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting ae39aec1940541fe9303fdd48819690c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731731144994 2024-11-16T04:25:52,478 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 874d68a0ba97439baeecae91e23b71c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731731147066 2024-11-16T04:25:52,478 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 460dbd12518f44e793dc1a280a384a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731731149226 2024-11-16T04:25:52,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742239_1415 (size=31869) 2024-11-16T04:25:52,485 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/79894e975ee54ef2850fc40004db4da2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/79894e975ee54ef2850fc40004db4da2 2024-11-16T04:25:52,488 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e2b8a1f3e14054afb97c94e793d2488#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:25:52,489 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/a0eca59881f04c63a8294180814c64a0 is 50, key is test_row_0/C:col10/1731731150376/Put/seqid=0 2024-11-16T04:25:52,490 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/A of 5e2b8a1f3e14054afb97c94e793d2488 into 79894e975ee54ef2850fc40004db4da2(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:52,490 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:52,490 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/A, priority=11, startTime=1731731152051; duration=0sec 2024-11-16T04:25:52,490 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:52,490 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:A 2024-11-16T04:25:52,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742240_1416 (size=12915) 2024-11-16T04:25:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:52,545 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:52,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:25:52,546 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-16T04:25:52,546 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:52,546 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,546 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,546 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:52,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e58417b965084f94985051462f1559e8_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_0/A:col10/1731731150392/Put/seqid=0 2024-11-16T04:25:52,554 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-16T04:25:52,554 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:52,554 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,554 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,554 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742241_1417 (size=12454) 2024-11-16T04:25:52,565 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-16T04:25:52,565 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-16T04:25:52,565 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:52,565 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:52,565 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,565 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,565 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,565 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:52,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:52,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:52,897 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/a0eca59881f04c63a8294180814c64a0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a0eca59881f04c63a8294180814c64a0 2024-11-16T04:25:52,901 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 5e2b8a1f3e14054afb97c94e793d2488/C of 5e2b8a1f3e14054afb97c94e793d2488 into a0eca59881f04c63a8294180814c64a0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:25:52,901 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:52,901 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., storeName=5e2b8a1f3e14054afb97c94e793d2488/C, priority=11, startTime=1731731152051; duration=0sec 2024-11-16T04:25:52,901 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:25:52,901 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e2b8a1f3e14054afb97c94e793d2488:C 2024-11-16T04:25:52,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:52,959 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e58417b965084f94985051462f1559e8_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e58417b965084f94985051462f1559e8_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:52,960 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/36ce145600e9449ebdc1c6c9813dfd8d, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:52,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/36ce145600e9449ebdc1c6c9813dfd8d is 175, key is test_row_0/A:col10/1731731150392/Put/seqid=0 2024-11-16T04:25:52,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742242_1418 (size=31255) 2024-11-16T04:25:53,368 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/36ce145600e9449ebdc1c6c9813dfd8d 2024-11-16T04:25:53,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/f7cee19cd8df4b1c9bb689e83939afc6 is 50, key is test_row_0/B:col10/1731731150392/Put/seqid=0 2024-11-16T04:25:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742243_1419 (size=12301) 2024-11-16T04:25:53,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/f7cee19cd8df4b1c9bb689e83939afc6 2024-11-16T04:25:53,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/b1a724ad11914bdb92aa180435ee02b9 is 50, key is test_row_0/C:col10/1731731150392/Put/seqid=0 2024-11-16T04:25:53,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742244_1420 (size=12301) 2024-11-16T04:25:54,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/b1a724ad11914bdb92aa180435ee02b9 2024-11-16T04:25:54,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/36ce145600e9449ebdc1c6c9813dfd8d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/36ce145600e9449ebdc1c6c9813dfd8d 2024-11-16T04:25:54,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/36ce145600e9449ebdc1c6c9813dfd8d, entries=150, sequenceid=288, filesize=30.5 K 2024-11-16T04:25:54,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/f7cee19cd8df4b1c9bb689e83939afc6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f7cee19cd8df4b1c9bb689e83939afc6 2024-11-16T04:25:54,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f7cee19cd8df4b1c9bb689e83939afc6, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T04:25:54,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/b1a724ad11914bdb92aa180435ee02b9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/b1a724ad11914bdb92aa180435ee02b9 2024-11-16T04:25:54,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/b1a724ad11914bdb92aa180435ee02b9, entries=150, sequenceid=288, filesize=12.0 K 2024-11-16T04:25:54,203 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=20.13 KB/20610 for 5e2b8a1f3e14054afb97c94e793d2488 in 1657ms, sequenceid=288, compaction requested=false 2024-11-16T04:25:54,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e2b8a1f3e14054afb97c94e793d2488: 2024-11-16T04:25:55,315 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-16T04:25:55,315 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:25:55,315 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:55,315 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:55,315 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:55,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T04:25:55,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-16T04:25:55,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-16T04:25:55,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-16T04:25:55,315 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1457 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4361 rows 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4373 rows 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4368 rows 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4374 rows 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1458 2024-11-16T04:25:55,316 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4372 rows 2024-11-16T04:25:55,316 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:25:55,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:25:55,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:55,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:55,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:25:55,317 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T04:25:55,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:55,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T04:25:55,321 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731155320"}]},"ts":"1731731155320"} 2024-11-16T04:25:55,322 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T04:25:55,322 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T04:25:55,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:25:55,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, UNASSIGN}] 2024-11-16T04:25:55,324 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, UNASSIGN 2024-11-16T04:25:55,325 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:55,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, UNASSIGN because future has completed 2024-11-16T04:25:55,327 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:25:55,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:55,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T04:25:55,479 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:55,479 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:25:55,480 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing 5e2b8a1f3e14054afb97c94e793d2488, disabling compactions & flushes 2024-11-16T04:25:55,480 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:55,480 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:55,480 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. after waiting 0 ms 2024-11-16T04:25:55,480 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:55,480 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing 5e2b8a1f3e14054afb97c94e793d2488 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-16T04:25:55,485 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c5cffd7c27b4725b2ebfb5afdbd2611_5e2b8a1f3e14054afb97c94e793d2488 is 50, key is test_row_1/A:col10/1731731152564/Put/seqid=0 2024-11-16T04:25:55,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742245_1421 (size=9914) 2024-11-16T04:25:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T04:25:55,902 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:55,905 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165c5cffd7c27b4725b2ebfb5afdbd2611_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c5cffd7c27b4725b2ebfb5afdbd2611_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:55,905 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/115d95c374da47389e06f841d4089314, store: [table=TestAcidGuarantees family=A region=5e2b8a1f3e14054afb97c94e793d2488] 2024-11-16T04:25:55,906 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/115d95c374da47389e06f841d4089314 is 175, key is test_row_1/A:col10/1731731152564/Put/seqid=0 2024-11-16T04:25:55,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742246_1422 (size=22561) 2024-11-16T04:25:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T04:25:56,310 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/115d95c374da47389e06f841d4089314 2024-11-16T04:25:56,315 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/eb11cb9ec23a430eb28f08f946a34070 is 50, key is test_row_1/B:col10/1731731152564/Put/seqid=0 2024-11-16T04:25:56,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742247_1423 (size=9857) 2024-11-16T04:25:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T04:25:56,719 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/eb11cb9ec23a430eb28f08f946a34070 2024-11-16T04:25:56,724 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/91e7b7ce585e4a3abf7d1e42b1f53e2f is 50, key is test_row_1/C:col10/1731731152564/Put/seqid=0 2024-11-16T04:25:56,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742248_1424 (size=9857) 2024-11-16T04:25:57,127 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/91e7b7ce585e4a3abf7d1e42b1f53e2f 2024-11-16T04:25:57,130 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/A/115d95c374da47389e06f841d4089314 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/115d95c374da47389e06f841d4089314 2024-11-16T04:25:57,133 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/115d95c374da47389e06f841d4089314, entries=100, sequenceid=296, filesize=22.0 K 2024-11-16T04:25:57,134 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/B/eb11cb9ec23a430eb28f08f946a34070 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/eb11cb9ec23a430eb28f08f946a34070 2024-11-16T04:25:57,137 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/eb11cb9ec23a430eb28f08f946a34070, entries=100, sequenceid=296, filesize=9.6 K 2024-11-16T04:25:57,138 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/.tmp/C/91e7b7ce585e4a3abf7d1e42b1f53e2f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/91e7b7ce585e4a3abf7d1e42b1f53e2f 2024-11-16T04:25:57,140 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/91e7b7ce585e4a3abf7d1e42b1f53e2f, entries=100, sequenceid=296, filesize=9.6 K 2024-11-16T04:25:57,141 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5e2b8a1f3e14054afb97c94e793d2488 in 1661ms, sequenceid=296, compaction requested=true 2024-11-16T04:25:57,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/6d0e7495bc904314a99c74af2ade845e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/d094a8e10bd64f80ab40c73ada2d2b2b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bcc3baa055e3497782b8559e3d387049, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/ed0aee43f4b84729b0917895f0638242, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3] to archive 2024-11-16T04:25:57,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:57,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/01c7a1e89e0a4243a8bf7d7cf728c5fd 2024-11-16T04:25:57,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/b57445b92fb342a7b3aeb68ee5d61e58 2024-11-16T04:25:57,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/811a71f020f942fd81f37b91d6503da7 2024-11-16T04:25:57,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/6d0e7495bc904314a99c74af2ade845e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/6d0e7495bc904314a99c74af2ade845e 2024-11-16T04:25:57,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/be1e4d61fa864659be8c7effc3820502 2024-11-16T04:25:57,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/319dd42f2915455d8d6ce9b07680c1ee 2024-11-16T04:25:57,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/d094a8e10bd64f80ab40c73ada2d2b2b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/d094a8e10bd64f80ab40c73ada2d2b2b 2024-11-16T04:25:57,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bb77777d3f0d402eaca794eff8905ba4 2024-11-16T04:25:57,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/e032285082ef44ffaded085010dce59e 2024-11-16T04:25:57,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bcc3baa055e3497782b8559e3d387049 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/bcc3baa055e3497782b8559e3d387049 2024-11-16T04:25:57,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/f0eb1cf51eff458fab79cee114c45f82 2024-11-16T04:25:57,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/c1b1ebf30bcf411ab27f2c830d910140 2024-11-16T04:25:57,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/0a226a0fc2fe4fccb1cded114f68c793 2024-11-16T04:25:57,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/ed0aee43f4b84729b0917895f0638242 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/ed0aee43f4b84729b0917895f0638242 2024-11-16T04:25:57,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/cfb2e48238c84b3097489d666f0fa55a 2024-11-16T04:25:57,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/1f7668f9695a4e119506d6bd3d6d7c30 2024-11-16T04:25:57,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/99d33bc992ce487b88c6256cb6edc598 2024-11-16T04:25:57,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/9a98dbb859a64bcdbb9f6bc01ed442c3 2024-11-16T04:25:57,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e806f354298c4b75945b79d9a0e5148a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/2a7583c97f764b519a3ff2b114bf3aa5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4629a10e52e64eb5a5d68acc933de648, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/042b88d3f5f14f0e8358b5f243c5f43d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/808b79d6d1274c84813ec62a1c5852db, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4f33562d11504fe2ab5ebd235af81696, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/0eea1a01866e47a7b61dc7bda6c600e9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/65ba650cc0004247b0310b6fc8138b35, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7f8a3598c2ad4667b21b0f6871d4d005, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7a3c9468cacd4d1b9cb132326bf20800, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e2f7853faef74f0abdede1a4fb32ca11, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/5988f8ad8a1f4fa09d0b549fc22b4c0a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/6513561447d943118bbe9175d74debea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/fd2816ea1297457b9a07c4649e4426b5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/55e87edc0d9142b1bbc57f4ba132581a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/468d98dfcb8d4464bba8d95fb451e72e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/a6fb2da1ae48447fa76abc7446d34a21, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/814b28a5966a47768bd855651597229c] to archive 2024-11-16T04:25:57,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:57,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e806f354298c4b75945b79d9a0e5148a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e806f354298c4b75945b79d9a0e5148a 2024-11-16T04:25:57,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/2a7583c97f764b519a3ff2b114bf3aa5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/2a7583c97f764b519a3ff2b114bf3aa5 2024-11-16T04:25:57,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4629a10e52e64eb5a5d68acc933de648 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4629a10e52e64eb5a5d68acc933de648 2024-11-16T04:25:57,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/042b88d3f5f14f0e8358b5f243c5f43d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/042b88d3f5f14f0e8358b5f243c5f43d 2024-11-16T04:25:57,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/808b79d6d1274c84813ec62a1c5852db to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/808b79d6d1274c84813ec62a1c5852db 2024-11-16T04:25:57,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4f33562d11504fe2ab5ebd235af81696 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/4f33562d11504fe2ab5ebd235af81696 2024-11-16T04:25:57,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/0eea1a01866e47a7b61dc7bda6c600e9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/0eea1a01866e47a7b61dc7bda6c600e9 2024-11-16T04:25:57,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/65ba650cc0004247b0310b6fc8138b35 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/65ba650cc0004247b0310b6fc8138b35 2024-11-16T04:25:57,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7f8a3598c2ad4667b21b0f6871d4d005 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7f8a3598c2ad4667b21b0f6871d4d005 2024-11-16T04:25:57,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7a3c9468cacd4d1b9cb132326bf20800 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/7a3c9468cacd4d1b9cb132326bf20800 2024-11-16T04:25:57,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e2f7853faef74f0abdede1a4fb32ca11 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/e2f7853faef74f0abdede1a4fb32ca11 2024-11-16T04:25:57,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/5988f8ad8a1f4fa09d0b549fc22b4c0a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/5988f8ad8a1f4fa09d0b549fc22b4c0a 2024-11-16T04:25:57,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/6513561447d943118bbe9175d74debea to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/6513561447d943118bbe9175d74debea 2024-11-16T04:25:57,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/fd2816ea1297457b9a07c4649e4426b5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/fd2816ea1297457b9a07c4649e4426b5 2024-11-16T04:25:57,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/55e87edc0d9142b1bbc57f4ba132581a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/55e87edc0d9142b1bbc57f4ba132581a 2024-11-16T04:25:57,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/468d98dfcb8d4464bba8d95fb451e72e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/468d98dfcb8d4464bba8d95fb451e72e 2024-11-16T04:25:57,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/a6fb2da1ae48447fa76abc7446d34a21 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/a6fb2da1ae48447fa76abc7446d34a21 2024-11-16T04:25:57,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/814b28a5966a47768bd855651597229c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/814b28a5966a47768bd855651597229c 2024-11-16T04:25:57,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7a33c2d4ce834ad88e27067523bf7678, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/942ac4b5732b4576a2f3be60f374ae9d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/80d916fbd7ba4c24840c16568e47f34e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a04fda3a408246bf97ca669f13dbc538, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/58a3954cbab64cbaa3a5a16c2a8d5bc8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/117a28b221fc42f5899e96c8be0ed2d7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/e585d2a7dac841b8b65d91b1f35adaac, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/694b936789c34b7d8796673bf92f5ad2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/9783f47ea76b4ceebbc6839bdd7e1a23, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/f024236593dc4f5f8ae2c9fc75ea80ce, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/3e20fb9ac4414966ad7b9461b94239ed, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/0101de8b7c8441dcb46b69915a7241b2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/19ec63690ee94186bb6c24d4af0078fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/12f38f164fcb48e4afc31937fdc54693, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7baf2c13a9474bcf8b90a04f55f85293, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/ae39aec1940541fe9303fdd48819690c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/874d68a0ba97439baeecae91e23b71c8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/460dbd12518f44e793dc1a280a384a23] to archive 2024-11-16T04:25:57,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:25:57,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7a33c2d4ce834ad88e27067523bf7678 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7a33c2d4ce834ad88e27067523bf7678 2024-11-16T04:25:57,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/942ac4b5732b4576a2f3be60f374ae9d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/942ac4b5732b4576a2f3be60f374ae9d 2024-11-16T04:25:57,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/80d916fbd7ba4c24840c16568e47f34e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/80d916fbd7ba4c24840c16568e47f34e 2024-11-16T04:25:57,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a04fda3a408246bf97ca669f13dbc538 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a04fda3a408246bf97ca669f13dbc538 2024-11-16T04:25:57,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/58a3954cbab64cbaa3a5a16c2a8d5bc8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/58a3954cbab64cbaa3a5a16c2a8d5bc8 2024-11-16T04:25:57,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/117a28b221fc42f5899e96c8be0ed2d7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/117a28b221fc42f5899e96c8be0ed2d7 2024-11-16T04:25:57,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/e585d2a7dac841b8b65d91b1f35adaac to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/e585d2a7dac841b8b65d91b1f35adaac 2024-11-16T04:25:57,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/694b936789c34b7d8796673bf92f5ad2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/694b936789c34b7d8796673bf92f5ad2 2024-11-16T04:25:57,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/9783f47ea76b4ceebbc6839bdd7e1a23 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/9783f47ea76b4ceebbc6839bdd7e1a23 2024-11-16T04:25:57,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/f024236593dc4f5f8ae2c9fc75ea80ce to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/f024236593dc4f5f8ae2c9fc75ea80ce 2024-11-16T04:25:57,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/3e20fb9ac4414966ad7b9461b94239ed to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/3e20fb9ac4414966ad7b9461b94239ed 2024-11-16T04:25:57,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/0101de8b7c8441dcb46b69915a7241b2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/0101de8b7c8441dcb46b69915a7241b2 2024-11-16T04:25:57,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/19ec63690ee94186bb6c24d4af0078fb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/19ec63690ee94186bb6c24d4af0078fb 2024-11-16T04:25:57,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/12f38f164fcb48e4afc31937fdc54693 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/12f38f164fcb48e4afc31937fdc54693 2024-11-16T04:25:57,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7baf2c13a9474bcf8b90a04f55f85293 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/7baf2c13a9474bcf8b90a04f55f85293 2024-11-16T04:25:57,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/ae39aec1940541fe9303fdd48819690c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/ae39aec1940541fe9303fdd48819690c 2024-11-16T04:25:57,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/874d68a0ba97439baeecae91e23b71c8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/874d68a0ba97439baeecae91e23b71c8 2024-11-16T04:25:57,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/460dbd12518f44e793dc1a280a384a23 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/460dbd12518f44e793dc1a280a384a23 2024-11-16T04:25:57,191 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/recovered.edits/299.seqid, newMaxSeqId=299, maxSeqId=4 2024-11-16T04:25:57,191 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. 2024-11-16T04:25:57,191 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for 5e2b8a1f3e14054afb97c94e793d2488: Waiting for close lock at 1731731155479Running coprocessor pre-close hooks at 1731731155479Disabling compacts and flushes for region at 1731731155479Disabling writes for close at 1731731155480 (+1 ms)Obtaining lock to block concurrent updates at 1731731155480Preparing flush snapshotting stores in 5e2b8a1f3e14054afb97c94e793d2488 at 1731731155480Finished memstore snapshotting TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731731155480Flushing stores of TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488. at 1731731155481 (+1 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/A: creating writer at 1731731155481Flushing mob file 5e2b8a1f3e14054afb97c94e793d2488/A: appending metadata at 1731731155485 (+4 ms)Flushing mob file 5e2b8a1f3e14054afb97c94e793d2488/A: closing flushed file at 1731731155485Flushing 5e2b8a1f3e14054afb97c94e793d2488/A: appending metadata at 1731731155905 (+420 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/A: closing flushed file at 1731731155906 (+1 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/B: creating writer at 1731731156313 (+407 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/B: appending metadata at 1731731156315 (+2 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/B: closing flushed file at 1731731156315Flushing 5e2b8a1f3e14054afb97c94e793d2488/C: creating writer at 1731731156722 (+407 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/C: appending metadata at 1731731156724 (+2 ms)Flushing 5e2b8a1f3e14054afb97c94e793d2488/C: closing flushed file at 1731731156724Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64158cba: reopening flushed file at 1731731157130 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a8c7063: reopening flushed file at 1731731157134 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd944dc: reopening flushed file at 1731731157137 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5e2b8a1f3e14054afb97c94e793d2488 in 1661ms, sequenceid=296, compaction requested=true at 1731731157141 (+4 ms)Writing region close event to WAL at 1731731157189 (+48 ms)Running coprocessor post-close hooks at 1731731157191 (+2 ms)Closed at 1731731157191 2024-11-16T04:25:57,193 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed 5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,195 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=5e2b8a1f3e14054afb97c94e793d2488, regionState=CLOSED 2024-11-16T04:25:57,195 WARN [regionserver/fda3efb342d8:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-16T04:25:57,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:57,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T04:25:57,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e2b8a1f3e14054afb97c94e793d2488, server=fda3efb342d8,45455,1731731034087 in 1.8690 sec 2024-11-16T04:25:57,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-16T04:25:57,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e2b8a1f3e14054afb97c94e793d2488, UNASSIGN in 1.8750 sec 2024-11-16T04:25:57,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T04:25:57,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-16T04:25:57,202 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731157202"}]},"ts":"1731731157202"} 2024-11-16T04:25:57,203 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T04:25:57,203 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T04:25:57,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8860 sec 2024-11-16T04:25:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T04:25:57,464 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:57,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T04:25:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T04:25:57,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,468 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,469 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/recovered.edits] 2024-11-16T04:25:57,471 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/115d95c374da47389e06f841d4089314 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/115d95c374da47389e06f841d4089314 2024-11-16T04:25:57,472 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/36ce145600e9449ebdc1c6c9813dfd8d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/36ce145600e9449ebdc1c6c9813dfd8d 2024-11-16T04:25:57,472 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/79894e975ee54ef2850fc40004db4da2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/A/79894e975ee54ef2850fc40004db4da2 2024-11-16T04:25:57,474 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/eb11cb9ec23a430eb28f08f946a34070 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/eb11cb9ec23a430eb28f08f946a34070 2024-11-16T04:25:57,475 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f03c011fedcf4c93a076a9fd9c6b2009 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f03c011fedcf4c93a076a9fd9c6b2009 2024-11-16T04:25:57,476 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f7cee19cd8df4b1c9bb689e83939afc6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/B/f7cee19cd8df4b1c9bb689e83939afc6 2024-11-16T04:25:57,477 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/91e7b7ce585e4a3abf7d1e42b1f53e2f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/91e7b7ce585e4a3abf7d1e42b1f53e2f 2024-11-16T04:25:57,478 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a0eca59881f04c63a8294180814c64a0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/a0eca59881f04c63a8294180814c64a0 2024-11-16T04:25:57,479 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/b1a724ad11914bdb92aa180435ee02b9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/C/b1a724ad11914bdb92aa180435ee02b9 2024-11-16T04:25:57,481 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/recovered.edits/299.seqid to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488/recovered.edits/299.seqid 2024-11-16T04:25:57,482 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,482 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T04:25:57,482 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T04:25:57,482 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T04:25:57,485 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ab9fe6dcae34a67baaa59b7ace938dd_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ab9fe6dcae34a67baaa59b7ace938dd_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,486 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ae8c67ba0f64eafa5011b9db060d5f0_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161ae8c67ba0f64eafa5011b9db060d5f0_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,486 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161bc5088c75c64258a100726b5234d5ce_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161bc5088c75c64258a100726b5234d5ce_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,487 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162f1483518878453fb016dc97cb492a04_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162f1483518878453fb016dc97cb492a04_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,488 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636e3e3c26e73409a858089a1167ffad1_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636e3e3c26e73409a858089a1167ffad1_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,489 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163738363d9e004301a93e679451f77d1a_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163738363d9e004301a93e679451f77d1a_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,490 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163b90f77c416b412196994db32f777687_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163b90f77c416b412196994db32f777687_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,491 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163d9d14f024f948c58fe1b8fbbc9b3cb0_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411163d9d14f024f948c58fe1b8fbbc9b3cb0_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,492 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164b70040fb7494ceeb0ed4ab04a2fab85_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164b70040fb7494ceeb0ed4ab04a2fab85_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,493 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c5cffd7c27b4725b2ebfb5afdbd2611_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165c5cffd7c27b4725b2ebfb5afdbd2611_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,494 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166116aaea12eb4a0383d0c0dd25f387f9_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166116aaea12eb4a0383d0c0dd25f387f9_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,495 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bbae9837c84a4e6488065ad9399a4e09_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116bbae9837c84a4e6488065ad9399a4e09_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,495 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d9d576e7e95541188ed1efbad6e42861_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d9d576e7e95541188ed1efbad6e42861_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,496 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e58417b965084f94985051462f1559e8_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e58417b965084f94985051462f1559e8_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,497 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e78952fec5df4e0f86a7e8665325c315_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e78952fec5df4e0f86a7e8665325c315_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,498 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eced906c2c854a89be5d3b344b21e479_5e2b8a1f3e14054afb97c94e793d2488 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eced906c2c854a89be5d3b344b21e479_5e2b8a1f3e14054afb97c94e793d2488 2024-11-16T04:25:57,498 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T04:25:57,500 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,502 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T04:25:57,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T04:25:57,504 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,504 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T04:25:57,505 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731731157504"}]},"ts":"9223372036854775807"} 2024-11-16T04:25:57,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T04:25:57,506 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5e2b8a1f3e14054afb97c94e793d2488, NAME => 'TestAcidGuarantees,,1731731128736.5e2b8a1f3e14054afb97c94e793d2488.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T04:25:57,506 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T04:25:57,506 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731731157506"}]},"ts":"9223372036854775807"} 2024-11-16T04:25:57,508 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T04:25:57,508 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-16T04:25:57,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T04:25:57,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T04:25:57,578 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:57,584 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 238) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/fda3efb342d8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1846551405_22 at /127.0.0.1:54968 [Waiting for operation #306] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1374812648_22 at /127.0.0.1:47432 [Waiting for operation #607] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 464), ProcessCount=11 (was 11), AvailableMemoryMB=8068 (was 8220) 2024-11-16T04:25:57,593 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=8068 2024-11-16T04:25:57,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:25:57,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:25:57,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T04:25:57,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T04:25:57,596 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:57,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-16T04:25:57,597 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T04:25:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T04:25:57,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742249_1425 (size=959) 2024-11-16T04:25:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T04:25:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T04:25:58,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:25:58,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742250_1426 (size=53) 2024-11-16T04:25:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T04:25:58,408 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:58,408 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3c5eac5e1b52fdf0c230183879dedd11, disabling compactions & flushes 2024-11-16T04:25:58,409 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. after waiting 0 ms 2024-11-16T04:25:58,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,409 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,409 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3c5eac5e1b52fdf0c230183879dedd11: Waiting for close lock at 1731731158408Disabling compacts and flushes for region at 1731731158408Disabling writes for close at 1731731158409 (+1 ms)Writing region close event to WAL at 1731731158409Closed at 1731731158409 2024-11-16T04:25:58,410 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T04:25:58,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731731158410"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731731158410"}]},"ts":"1731731158410"} 2024-11-16T04:25:58,411 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T04:25:58,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T04:25:58,412 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731158411"}]},"ts":"1731731158411"} 2024-11-16T04:25:58,412 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T04:25:58,413 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, ASSIGN}] 2024-11-16T04:25:58,414 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, ASSIGN 2024-11-16T04:25:58,414 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, ASSIGN; state=OFFLINE, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=false 2024-11-16T04:25:58,565 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3c5eac5e1b52fdf0c230183879dedd11, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:58,566 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, ASSIGN because future has completed 2024-11-16T04:25:58,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:25:58,721 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,721 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:25:58,721 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,722 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:25:58,722 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,722 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,723 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,724 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c5eac5e1b52fdf0c230183879dedd11 columnFamilyName A 2024-11-16T04:25:58,724 DEBUG [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:58,724 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] regionserver.HStore(327): Store=3c5eac5e1b52fdf0c230183879dedd11/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:58,724 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,725 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c5eac5e1b52fdf0c230183879dedd11 columnFamilyName B 2024-11-16T04:25:58,725 DEBUG [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:58,725 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] regionserver.HStore(327): Store=3c5eac5e1b52fdf0c230183879dedd11/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:58,726 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,726 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c5eac5e1b52fdf0c230183879dedd11 columnFamilyName C 2024-11-16T04:25:58,726 DEBUG [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:25:58,727 INFO [StoreOpener-3c5eac5e1b52fdf0c230183879dedd11-1 {}] regionserver.HStore(327): Store=3c5eac5e1b52fdf0c230183879dedd11/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:25:58,727 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,727 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,728 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,728 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,728 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,729 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:25:58,729 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,731 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:25:58,731 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 3c5eac5e1b52fdf0c230183879dedd11; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68826718, jitterRate=0.02559801936149597}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:25:58,731 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:58,732 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 3c5eac5e1b52fdf0c230183879dedd11: Running coprocessor pre-open hook at 1731731158722Writing region info on filesystem at 1731731158722Initializing all the Stores at 1731731158722Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731158722Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731158723 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731158723Cleaning up temporary data from old regions at 1731731158728 (+5 ms)Running coprocessor post-open hooks at 1731731158731 (+3 ms)Region opened successfully at 1731731158732 (+1 ms) 2024-11-16T04:25:58,732 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., pid=114, masterSystemTime=1731731158718 2024-11-16T04:25:58,733 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,733 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T04:25:58,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=3c5eac5e1b52fdf0c230183879dedd11, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:58,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:25:58,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T04:25:58,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 in 170 msec 2024-11-16T04:25:58,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-16T04:25:58,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, ASSIGN in 324 msec 2024-11-16T04:25:58,740 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T04:25:58,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731158740"}]},"ts":"1731731158740"} 2024-11-16T04:25:58,741 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T04:25:58,742 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T04:25:58,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-16T04:25:59,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-16T04:25:59,743 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:25:59,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6163d4d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d13b3f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@686db442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,748 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,749 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,751 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:25:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:25:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2c53366b 2024-11-16T04:25:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:25:59,752 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:25:59,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac1b790, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,754 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70f58044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e76276b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe58708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4284adda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36fb05ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a996003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,761 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65037512, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,762 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74eccf31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a177044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,764 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b1dfe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,765 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b1f347, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbbba3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,766 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a8cf28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e975318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b052618, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,769 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca825db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,770 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ede8f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48bc9769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efa80b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4564c544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460a8c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4737f29e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@124ba7f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b76d744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,778 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b75c9d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,779 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4e42a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694dc39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,780 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:25:59,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:25:59,781 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:25:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:25:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:25:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161dea0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:25:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:25:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:25:59,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:25:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2c3930, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:25:59,783 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:25:59,783 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,783 DEBUG [Thread-1921 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,783 DEBUG [Thread-1919 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,784 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,784 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-16T04:25:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:25:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,785 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,785 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:25:59,786 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,786 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,786 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:25:59,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:25:59,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:25:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:25:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:25:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,794 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2] 2024-11-16T04:25:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:25:59,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:25:59,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:25:59,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/3aabb17fa608407d983e22782cf80095 is 50, key is test_row_0/A:col10/1731731159800/Put/seqid=0 2024-11-16T04:25:59,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742251_1427 (size=14341) 2024-11-16T04:25:59,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/3aabb17fa608407d983e22782cf80095 2024-11-16T04:25:59,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731219848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731219857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731219857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731219858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731219859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/22b422b1392541c484490c29c5b8b6e0 is 50, key is test_row_0/B:col10/1731731159800/Put/seqid=0 2024-11-16T04:25:59,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742252_1428 (size=12001) 2024-11-16T04:25:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:25:59,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:25:59,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:59,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:25:59,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:25:59,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:59,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:59,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:25:59,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731219963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731219973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731219973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731219973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:25:59,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:25:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731219983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:25:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:25:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:26:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:00,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:26:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731220173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731220183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731220184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731220194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731220194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:26:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/22b422b1392541c484490c29c5b8b6e0 2024-11-16T04:26:00,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/2b5869e3b8aa4cd0bee2ab2699da6296 is 50, key is test_row_0/C:col10/1731731159800/Put/seqid=0 2024-11-16T04:26:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742253_1429 (size=12001) 2024-11-16T04:26:00,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:26:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:26:00,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731220494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731220494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731220504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731220504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731220504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:00,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:26:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:26:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:00,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:00,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/2b5869e3b8aa4cd0bee2ab2699da6296 2024-11-16T04:26:00,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/3aabb17fa608407d983e22782cf80095 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3aabb17fa608407d983e22782cf80095 2024-11-16T04:26:00,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3aabb17fa608407d983e22782cf80095, entries=200, sequenceid=13, filesize=14.0 K 2024-11-16T04:26:00,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/22b422b1392541c484490c29c5b8b6e0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/22b422b1392541c484490c29c5b8b6e0 2024-11-16T04:26:00,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/22b422b1392541c484490c29c5b8b6e0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T04:26:00,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/2b5869e3b8aa4cd0bee2ab2699da6296 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2b5869e3b8aa4cd0bee2ab2699da6296 2024-11-16T04:26:00,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2b5869e3b8aa4cd0bee2ab2699da6296, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T04:26:00,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3c5eac5e1b52fdf0c230183879dedd11 in 971ms, sequenceid=13, compaction requested=false 2024-11-16T04:26:00,771 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T04:26:00,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:00,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T04:26:00,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:00,858 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:26:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/13ec4a7348a64925910952992c206abc is 50, key is test_row_0/A:col10/1731731159857/Put/seqid=0 2024-11-16T04:26:00,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742254_1430 (size=12001) 2024-11-16T04:26:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:26:01,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:01,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:01,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731221023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731221024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731221025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731221026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731221027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731221133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731221133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731221143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731221144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/13ec4a7348a64925910952992c206abc 2024-11-16T04:26:01,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/d8e4fd87d7b244faab24de50315b84b9 is 50, key is test_row_0/B:col10/1731731159857/Put/seqid=0 2024-11-16T04:26:01,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742255_1431 (size=12001) 2024-11-16T04:26:01,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731221343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731221343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731221353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731221354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731221653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731221653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731221663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731221663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:01,674 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/d8e4fd87d7b244faab24de50315b84b9 2024-11-16T04:26:01,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/292c14072da4436d8778209464b7ad8f is 50, key is test_row_0/C:col10/1731731159857/Put/seqid=0 2024-11-16T04:26:01,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742256_1432 (size=12001) 2024-11-16T04:26:01,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:26:02,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731222033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,041 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,083 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/292c14072da4436d8778209464b7ad8f 2024-11-16T04:26:02,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/13ec4a7348a64925910952992c206abc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/13ec4a7348a64925910952992c206abc 2024-11-16T04:26:02,096 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/13ec4a7348a64925910952992c206abc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T04:26:02,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/d8e4fd87d7b244faab24de50315b84b9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/d8e4fd87d7b244faab24de50315b84b9 2024-11-16T04:26:02,101 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/d8e4fd87d7b244faab24de50315b84b9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T04:26:02,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/292c14072da4436d8778209464b7ad8f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/292c14072da4436d8778209464b7ad8f 2024-11-16T04:26:02,107 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/292c14072da4436d8778209464b7ad8f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-16T04:26:02,108 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 3c5eac5e1b52fdf0c230183879dedd11 in 1250ms, sequenceid=37, compaction requested=false 2024-11-16T04:26:02,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:02,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:02,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-16T04:26:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-16T04:26:02,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T04:26:02,112 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-16T04:26:02,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.3290 sec 2024-11-16T04:26:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:02,168 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:02,168 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T04:26:02,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/a28a7b9b54c44ac28394cc660e3cca24 is 50, key is test_row_0/A:col10/1731731161016/Put/seqid=0 2024-11-16T04:26:02,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742257_1433 (size=14341) 2024-11-16T04:26:02,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731222223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731222227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731222228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731222229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731222343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731222343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731222343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731222343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731222553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731222554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731222554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731222554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/a28a7b9b54c44ac28394cc660e3cca24 2024-11-16T04:26:02,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/5b3c564ce5d0488ca2b41cd0af05d005 is 50, key is test_row_0/B:col10/1731731161016/Put/seqid=0 2024-11-16T04:26:02,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742258_1434 (size=12001) 2024-11-16T04:26:02,710 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:26:02,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731222863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731222874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731222874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731222874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:02,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/5b3c564ce5d0488ca2b41cd0af05d005 2024-11-16T04:26:03,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d706972f18bd4afabe8e766aaac6d6c4 is 50, key is test_row_0/C:col10/1731731161016/Put/seqid=0 2024-11-16T04:26:03,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742259_1435 (size=12001) 2024-11-16T04:26:03,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d706972f18bd4afabe8e766aaac6d6c4 2024-11-16T04:26:03,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/a28a7b9b54c44ac28394cc660e3cca24 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/a28a7b9b54c44ac28394cc660e3cca24 2024-11-16T04:26:03,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/a28a7b9b54c44ac28394cc660e3cca24, entries=200, sequenceid=51, filesize=14.0 K 2024-11-16T04:26:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/5b3c564ce5d0488ca2b41cd0af05d005 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/5b3c564ce5d0488ca2b41cd0af05d005 2024-11-16T04:26:03,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/5b3c564ce5d0488ca2b41cd0af05d005, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T04:26:03,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d706972f18bd4afabe8e766aaac6d6c4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d706972f18bd4afabe8e766aaac6d6c4 2024-11-16T04:26:03,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d706972f18bd4afabe8e766aaac6d6c4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T04:26:03,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c5eac5e1b52fdf0c230183879dedd11 in 997ms, sequenceid=51, compaction requested=true 2024-11-16T04:26:03,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:03,166 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:03,166 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:03,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:03,166 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:03,167 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:03,167 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:03,167 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3aabb17fa608407d983e22782cf80095, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/13ec4a7348a64925910952992c206abc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/a28a7b9b54c44ac28394cc660e3cca24] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=39.7 K 2024-11-16T04:26:03,167 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:03,167 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:03,167 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aabb17fa608407d983e22782cf80095, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731159795 2024-11-16T04:26:03,167 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:03,167 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/22b422b1392541c484490c29c5b8b6e0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/d8e4fd87d7b244faab24de50315b84b9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/5b3c564ce5d0488ca2b41cd0af05d005] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=35.2 K 2024-11-16T04:26:03,167 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13ec4a7348a64925910952992c206abc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731731159819 2024-11-16T04:26:03,168 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b422b1392541c484490c29c5b8b6e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731159796 2024-11-16T04:26:03,168 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28a7b9b54c44ac28394cc660e3cca24, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731161016 2024-11-16T04:26:03,168 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d8e4fd87d7b244faab24de50315b84b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731731159819 2024-11-16T04:26:03,168 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b3c564ce5d0488ca2b41cd0af05d005, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731161016 2024-11-16T04:26:03,187 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:03,188 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/b2760d7ff0934819b486894252980d2d is 50, key is test_row_0/A:col10/1731731161016/Put/seqid=0 2024-11-16T04:26:03,190 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:03,191 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/b451306883614f0aa476dff908afc658 is 50, key is test_row_0/B:col10/1731731161016/Put/seqid=0 2024-11-16T04:26:03,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742260_1436 (size=12104) 2024-11-16T04:26:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742261_1437 (size=12104) 2024-11-16T04:26:03,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:03,388 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:03,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:26:03,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/603eafdb7695460891ab0704ccbed61c is 50, key is test_row_0/A:col10/1731731162222/Put/seqid=0 2024-11-16T04:26:03,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731223426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731223435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742262_1438 (size=14341) 2024-11-16T04:26:03,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/603eafdb7695460891ab0704ccbed61c 2024-11-16T04:26:03,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731223437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731223440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/4593f6e719094f57953f9efc3b186de2 is 50, key is test_row_0/B:col10/1731731162222/Put/seqid=0 2024-11-16T04:26:03,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742263_1439 (size=12001) 2024-11-16T04:26:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731223553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731223553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731223564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731223564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,628 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/b2760d7ff0934819b486894252980d2d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/b2760d7ff0934819b486894252980d2d 2024-11-16T04:26:03,634 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/b451306883614f0aa476dff908afc658 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b451306883614f0aa476dff908afc658 2024-11-16T04:26:03,639 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into b2760d7ff0934819b486894252980d2d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:03,640 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:03,640 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731163165; duration=0sec 2024-11-16T04:26:03,640 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:03,640 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:03,640 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:03,645 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into b451306883614f0aa476dff908afc658(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:03,645 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:03,645 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731163166; duration=0sec 2024-11-16T04:26:03,645 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:03,645 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:03,646 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:03,646 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:03,646 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:03,646 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2b5869e3b8aa4cd0bee2ab2699da6296, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/292c14072da4436d8778209464b7ad8f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d706972f18bd4afabe8e766aaac6d6c4] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=35.2 K 2024-11-16T04:26:03,646 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5869e3b8aa4cd0bee2ab2699da6296, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731731159796 2024-11-16T04:26:03,647 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 292c14072da4436d8778209464b7ad8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731731159819 2024-11-16T04:26:03,647 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d706972f18bd4afabe8e766aaac6d6c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731161016 2024-11-16T04:26:03,670 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:03,671 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/3905320817fa4447977d88b6abd46c47 is 50, key is test_row_0/C:col10/1731731161016/Put/seqid=0 2024-11-16T04:26:03,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742264_1440 (size=12104) 2024-11-16T04:26:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731223774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731223774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731223785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:03,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731223786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:03,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:03,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/4593f6e719094f57953f9efc3b186de2 2024-11-16T04:26:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T04:26:03,945 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:03,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:03,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T04:26:03,947 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T04:26:03,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:03,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:03,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/000a4f9db40949ac9828893d1e476383 is 50, key is test_row_0/C:col10/1731731162222/Put/seqid=0 2024-11-16T04:26:04,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742265_1441 (size=12001) 2024-11-16T04:26:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T04:26:04,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731224063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,069 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731224094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731224094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T04:26:04,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731224094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731224114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,123 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,124 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/3905320817fa4447977d88b6abd46c47 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3905320817fa4447977d88b6abd46c47 2024-11-16T04:26:04,137 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 3905320817fa4447977d88b6abd46c47(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:04,137 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:04,137 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731163166; duration=0sec 2024-11-16T04:26:04,137 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:04,137 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:04,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T04:26:04,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:04,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T04:26:04,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/000a4f9db40949ac9828893d1e476383 2024-11-16T04:26:04,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T04:26:04,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:04,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/603eafdb7695460891ab0704ccbed61c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/603eafdb7695460891ab0704ccbed61c 2024-11-16T04:26:04,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/603eafdb7695460891ab0704ccbed61c, entries=200, sequenceid=74, filesize=14.0 K 2024-11-16T04:26:04,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/4593f6e719094f57953f9efc3b186de2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/4593f6e719094f57953f9efc3b186de2 2024-11-16T04:26:04,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/4593f6e719094f57953f9efc3b186de2, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T04:26:04,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/000a4f9db40949ac9828893d1e476383 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/000a4f9db40949ac9828893d1e476383 2024-11-16T04:26:04,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/000a4f9db40949ac9828893d1e476383, entries=150, sequenceid=74, filesize=11.7 K 2024-11-16T04:26:04,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3c5eac5e1b52fdf0c230183879dedd11 in 1050ms, sequenceid=74, compaction requested=false 2024-11-16T04:26:04,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:04,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T04:26:04,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,566 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:26:04,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/8190fed7a7c2414598bb558c152db1b1 is 50, key is test_row_0/A:col10/1731731163431/Put/seqid=0 2024-11-16T04:26:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T04:26:04,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742266_1442 (size=12001) 2024-11-16T04:26:04,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/8190fed7a7c2414598bb558c152db1b1 2024-11-16T04:26:04,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/c5137fc286574863b2d2404ff1c5365c is 50, key is test_row_0/B:col10/1731731163431/Put/seqid=0 2024-11-16T04:26:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:04,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:04,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742267_1443 (size=12001) 2024-11-16T04:26:04,618 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/c5137fc286574863b2d2404ff1c5365c 2024-11-16T04:26:04,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/668820a842d9408aa0aaea4e2203d759 is 50, key is test_row_0/C:col10/1731731163431/Put/seqid=0 2024-11-16T04:26:04,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742268_1444 (size=12001) 2024-11-16T04:26:04,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/668820a842d9408aa0aaea4e2203d759 2024-11-16T04:26:04,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/8190fed7a7c2414598bb558c152db1b1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8190fed7a7c2414598bb558c152db1b1 2024-11-16T04:26:04,676 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8190fed7a7c2414598bb558c152db1b1, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T04:26:04,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/c5137fc286574863b2d2404ff1c5365c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c5137fc286574863b2d2404ff1c5365c 2024-11-16T04:26:04,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731224672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731224672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731224682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,688 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c5137fc286574863b2d2404ff1c5365c, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T04:26:04,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/668820a842d9408aa0aaea4e2203d759 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/668820a842d9408aa0aaea4e2203d759 2024-11-16T04:26:04,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731224685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/668820a842d9408aa0aaea4e2203d759, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T04:26:04,711 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 3c5eac5e1b52fdf0c230183879dedd11 in 144ms, sequenceid=90, compaction requested=true 2024-11-16T04:26:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:04,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T04:26:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T04:26:04,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T04:26:04,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 764 msec 2024-11-16T04:26:04,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 769 msec 2024-11-16T04:26:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:04,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:26:04,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/90349fac7460495eab68dce41ad51ce2 is 50, key is test_row_0/A:col10/1731731164667/Put/seqid=0 2024-11-16T04:26:04,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742269_1445 (size=14341) 2024-11-16T04:26:04,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731224810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731224814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731224818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731224819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731224923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731224934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731224934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:04,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:04,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731224943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:04,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T04:26:05,083 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:05,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-16T04:26:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T04:26:05,094 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:05,094 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:05,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:05,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731225134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731225143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731225143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731225163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T04:26:05,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/90349fac7460495eab68dce41ad51ce2 2024-11-16T04:26:05,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/a0de66effe8f4c658f681e0d690799cf is 50, key is test_row_0/B:col10/1731731164667/Put/seqid=0 2024-11-16T04:26:05,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T04:26:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:05,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:05,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:05,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742270_1446 (size=12001) 2024-11-16T04:26:05,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/a0de66effe8f4c658f681e0d690799cf 2024-11-16T04:26:05,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/a4dec5a7107d4636848d4e2950384a6f is 50, key is test_row_0/C:col10/1731731164667/Put/seqid=0 2024-11-16T04:26:05,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742271_1447 (size=12001) 2024-11-16T04:26:05,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/a4dec5a7107d4636848d4e2950384a6f 2024-11-16T04:26:05,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/90349fac7460495eab68dce41ad51ce2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/90349fac7460495eab68dce41ad51ce2 2024-11-16T04:26:05,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/90349fac7460495eab68dce41ad51ce2, entries=200, sequenceid=115, filesize=14.0 K 2024-11-16T04:26:05,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/a0de66effe8f4c658f681e0d690799cf as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a0de66effe8f4c658f681e0d690799cf 2024-11-16T04:26:05,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a0de66effe8f4c658f681e0d690799cf, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:26:05,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/a4dec5a7107d4636848d4e2950384a6f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/a4dec5a7107d4636848d4e2950384a6f 2024-11-16T04:26:05,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/a4dec5a7107d4636848d4e2950384a6f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-16T04:26:05,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3c5eac5e1b52fdf0c230183879dedd11 in 522ms, sequenceid=115, compaction requested=true 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:05,319 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:05,319 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:05,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:05,320 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:05,320 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:05,320 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:05,320 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:05,320 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:05,320 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:05,320 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/b2760d7ff0934819b486894252980d2d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/603eafdb7695460891ab0704ccbed61c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8190fed7a7c2414598bb558c152db1b1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/90349fac7460495eab68dce41ad51ce2] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=51.5 K 2024-11-16T04:26:05,320 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b451306883614f0aa476dff908afc658, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/4593f6e719094f57953f9efc3b186de2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c5137fc286574863b2d2404ff1c5365c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a0de66effe8f4c658f681e0d690799cf] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=47.0 K 2024-11-16T04:26:05,320 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b451306883614f0aa476dff908afc658, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731161016 2024-11-16T04:26:05,320 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2760d7ff0934819b486894252980d2d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731161016 2024-11-16T04:26:05,321 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 603eafdb7695460891ab0704ccbed61c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731731162222 2024-11-16T04:26:05,321 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4593f6e719094f57953f9efc3b186de2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731731162222 2024-11-16T04:26:05,321 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c5137fc286574863b2d2404ff1c5365c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731731163424 2024-11-16T04:26:05,321 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8190fed7a7c2414598bb558c152db1b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731731163424 2024-11-16T04:26:05,321 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a0de66effe8f4c658f681e0d690799cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731164667 2024-11-16T04:26:05,321 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90349fac7460495eab68dce41ad51ce2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731164667 2024-11-16T04:26:05,329 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:05,330 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/0791d03dfe5f4c338a961f683c95eac6 is 50, key is test_row_0/A:col10/1731731164667/Put/seqid=0 2024-11-16T04:26:05,332 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:05,333 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8df3242421a1415c8217128ab5cac953 is 50, key is test_row_0/B:col10/1731731164667/Put/seqid=0 2024-11-16T04:26:05,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742272_1448 (size=12241) 2024-11-16T04:26:05,375 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8df3242421a1415c8217128ab5cac953 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8df3242421a1415c8217128ab5cac953 2024-11-16T04:26:05,388 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 8df3242421a1415c8217128ab5cac953(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:05,388 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:05,388 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=12, startTime=1731731165319; duration=0sec 2024-11-16T04:26:05,388 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:05,388 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:05,389 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:05,392 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:05,392 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:05,392 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:05,392 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3905320817fa4447977d88b6abd46c47, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/000a4f9db40949ac9828893d1e476383, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/668820a842d9408aa0aaea4e2203d759, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/a4dec5a7107d4636848d4e2950384a6f] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=47.0 K 2024-11-16T04:26:05,392 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3905320817fa4447977d88b6abd46c47, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731731161016 2024-11-16T04:26:05,393 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 000a4f9db40949ac9828893d1e476383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731731162222 2024-11-16T04:26:05,393 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 668820a842d9408aa0aaea4e2203d759, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731731163424 2024-11-16T04:26:05,394 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a4dec5a7107d4636848d4e2950384a6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731164667 2024-11-16T04:26:05,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742273_1449 (size=12241) 2024-11-16T04:26:05,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-16T04:26:05,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T04:26:05,404 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:26:05,412 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/0791d03dfe5f4c338a961f683c95eac6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/0791d03dfe5f4c338a961f683c95eac6 2024-11-16T04:26:05,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/9c41e659a37e47a6b789f6890b19e047 is 50, key is test_row_0/A:col10/1731731164817/Put/seqid=0 2024-11-16T04:26:05,425 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into 0791d03dfe5f4c338a961f683c95eac6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:05,425 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:05,425 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=12, startTime=1731731165319; duration=0sec 2024-11-16T04:26:05,425 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:05,425 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:05,431 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:05,431 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/9990dcfa024a4b8fa6270c5887c94fdd is 50, key is test_row_0/C:col10/1731731164667/Put/seqid=0 2024-11-16T04:26:05,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742274_1450 (size=12001) 2024-11-16T04:26:05,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/9c41e659a37e47a6b789f6890b19e047 2024-11-16T04:26:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:05,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:05,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742275_1451 (size=12241) 2024-11-16T04:26:05,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/934297517b9d4bd481ef8652cb9d2636 is 50, key is test_row_0/B:col10/1731731164817/Put/seqid=0 2024-11-16T04:26:05,466 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/9990dcfa024a4b8fa6270c5887c94fdd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/9990dcfa024a4b8fa6270c5887c94fdd 2024-11-16T04:26:05,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742276_1452 (size=12001) 2024-11-16T04:26:05,473 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 9990dcfa024a4b8fa6270c5887c94fdd(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:05,473 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:05,473 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=12, startTime=1731731165319; duration=0sec 2024-11-16T04:26:05,473 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:05,473 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:05,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731225545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731225547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731225561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731225562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731225674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731225674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731225685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731225689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T04:26:05,870 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/934297517b9d4bd481ef8652cb9d2636 2024-11-16T04:26:05,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d7e7f511316948a0991c8de030e6fc4f is 50, key is test_row_0/C:col10/1731731164817/Put/seqid=0 2024-11-16T04:26:05,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742277_1453 (size=12001) 2024-11-16T04:26:05,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731225883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731225883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731225903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731225903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:05,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731226203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731226203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T04:26:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731226224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731226224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,232 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,283 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d7e7f511316948a0991c8de030e6fc4f 2024-11-16T04:26:06,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/9c41e659a37e47a6b789f6890b19e047 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9c41e659a37e47a6b789f6890b19e047 2024-11-16T04:26:06,291 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9c41e659a37e47a6b789f6890b19e047, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T04:26:06,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/934297517b9d4bd481ef8652cb9d2636 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/934297517b9d4bd481ef8652cb9d2636 2024-11-16T04:26:06,302 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/934297517b9d4bd481ef8652cb9d2636, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T04:26:06,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d7e7f511316948a0991c8de030e6fc4f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d7e7f511316948a0991c8de030e6fc4f 2024-11-16T04:26:06,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d7e7f511316948a0991c8de030e6fc4f, entries=150, sequenceid=127, filesize=11.7 K 2024-11-16T04:26:06,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3c5eac5e1b52fdf0c230183879dedd11 in 904ms, sequenceid=127, compaction requested=false 2024-11-16T04:26:06,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:06,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:06,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-16T04:26:06,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-16T04:26:06,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T04:26:06,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2180 sec 2024-11-16T04:26:06,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2330 sec 2024-11-16T04:26:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:06,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:06,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T04:26:06,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6f86a75bf9d947638bdb9bcb922233b7 is 50, key is test_row_0/A:col10/1731731165560/Put/seqid=0 2024-11-16T04:26:06,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742278_1454 (size=16931) 2024-11-16T04:26:06,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6f86a75bf9d947638bdb9bcb922233b7 2024-11-16T04:26:06,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/95117a10737147b9a9a8d8e806747235 is 50, key is test_row_0/B:col10/1731731165560/Put/seqid=0 2024-11-16T04:26:06,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731226741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731226741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742279_1455 (size=12151) 2024-11-16T04:26:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731226744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731226746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731226854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731226856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731226863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731226863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731227064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731227074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731227074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731227074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/95117a10737147b9a9a8d8e806747235 2024-11-16T04:26:07,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/272c1407e3bb49d7bd26e76ab5da58cd is 50, key is test_row_0/C:col10/1731731165560/Put/seqid=0 2024-11-16T04:26:07,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742280_1456 (size=12151) 2024-11-16T04:26:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T04:26:07,234 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:07,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-16T04:26:07,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T04:26:07,235 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:07,236 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:07,236 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T04:26:07,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731227374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T04:26:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:07,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731227393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731227393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731227393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,402 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T04:26:07,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:07,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:07,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:07,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T04:26:07,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/272c1407e3bb49d7bd26e76ab5da58cd 2024-11-16T04:26:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6f86a75bf9d947638bdb9bcb922233b7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6f86a75bf9d947638bdb9bcb922233b7 2024-11-16T04:26:07,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6f86a75bf9d947638bdb9bcb922233b7, entries=250, sequenceid=156, filesize=16.5 K 2024-11-16T04:26:07,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/95117a10737147b9a9a8d8e806747235 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/95117a10737147b9a9a8d8e806747235 2024-11-16T04:26:07,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/95117a10737147b9a9a8d8e806747235, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T04:26:07,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/272c1407e3bb49d7bd26e76ab5da58cd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/272c1407e3bb49d7bd26e76ab5da58cd 2024-11-16T04:26:07,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/272c1407e3bb49d7bd26e76ab5da58cd, entries=150, sequenceid=156, filesize=11.9 K 2024-11-16T04:26:07,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 3c5eac5e1b52fdf0c230183879dedd11 in 869ms, sequenceid=156, compaction requested=true 2024-11-16T04:26:07,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:07,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:07,599 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:07,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:07,599 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:07,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:07,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:07,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:07,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:07,600 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41173 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:07,600 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:07,600 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,600 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/0791d03dfe5f4c338a961f683c95eac6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9c41e659a37e47a6b789f6890b19e047, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6f86a75bf9d947638bdb9bcb922233b7] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=40.2 K 2024-11-16T04:26:07,600 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:07,600 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0791d03dfe5f4c338a961f683c95eac6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731164667 2024-11-16T04:26:07,600 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:07,600 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,601 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8df3242421a1415c8217128ab5cac953, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/934297517b9d4bd481ef8652cb9d2636, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/95117a10737147b9a9a8d8e806747235] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=35.5 K 2024-11-16T04:26:07,601 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c41e659a37e47a6b789f6890b19e047, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731731164812 2024-11-16T04:26:07,601 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df3242421a1415c8217128ab5cac953, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731164667 2024-11-16T04:26:07,601 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f86a75bf9d947638bdb9bcb922233b7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731731165520 2024-11-16T04:26:07,602 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 934297517b9d4bd481ef8652cb9d2636, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731731164812 2024-11-16T04:26:07,602 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 95117a10737147b9a9a8d8e806747235, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731731165558 2024-11-16T04:26:07,613 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:07,613 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/7aecb901fd604d2e9f97dd7bcda00bcf is 50, key is test_row_0/A:col10/1731731165560/Put/seqid=0 2024-11-16T04:26:07,616 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:07,616 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8757205544464cc5a582ef9744674701 is 50, key is test_row_0/B:col10/1731731165560/Put/seqid=0 2024-11-16T04:26:07,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742282_1458 (size=12493) 2024-11-16T04:26:07,638 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8757205544464cc5a582ef9744674701 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8757205544464cc5a582ef9744674701 2024-11-16T04:26:07,642 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 8757205544464cc5a582ef9744674701(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:07,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:07,643 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731167599; duration=0sec 2024-11-16T04:26:07,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:07,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:07,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:07,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:07,643 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:07,643 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,644 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/9990dcfa024a4b8fa6270c5887c94fdd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d7e7f511316948a0991c8de030e6fc4f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/272c1407e3bb49d7bd26e76ab5da58cd] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=35.5 K 2024-11-16T04:26:07,644 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9990dcfa024a4b8fa6270c5887c94fdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731731164667 2024-11-16T04:26:07,644 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e7f511316948a0991c8de030e6fc4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731731164812 2024-11-16T04:26:07,645 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 272c1407e3bb49d7bd26e76ab5da58cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731731165558 2024-11-16T04:26:07,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742281_1457 (size=12493) 2024-11-16T04:26:07,653 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:07,654 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/6be8c97fc4f34c1993f08b5fba06dab1 is 50, key is test_row_0/C:col10/1731731165560/Put/seqid=0 2024-11-16T04:26:07,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742283_1459 (size=12493) 2024-11-16T04:26:07,677 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/6be8c97fc4f34c1993f08b5fba06dab1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/6be8c97fc4f34c1993f08b5fba06dab1 2024-11-16T04:26:07,680 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 6be8c97fc4f34c1993f08b5fba06dab1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:07,680 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:07,680 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731167599; duration=0sec 2024-11-16T04:26:07,680 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:07,680 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:07,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-16T04:26:07,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:07,701 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-16T04:26:07,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/dbb9cff47b3e4686b698c3433947f7f8 is 50, key is test_row_1/A:col10/1731731166744/Put/seqid=0 2024-11-16T04:26:07,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742284_1460 (size=9757) 2024-11-16T04:26:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T04:26:07,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:07,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731227963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731227963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731227964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731227965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,052 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/7aecb901fd604d2e9f97dd7bcda00bcf as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/7aecb901fd604d2e9f97dd7bcda00bcf 2024-11-16T04:26:08,055 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into 7aecb901fd604d2e9f97dd7bcda00bcf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:08,055 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:08,055 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731167599; duration=0sec 2024-11-16T04:26:08,055 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:08,055 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731228073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731228073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731228083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731228084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,118 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/dbb9cff47b3e4686b698c3433947f7f8 2024-11-16T04:26:08,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731228115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,124 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/1900d42858fe47718962bff7d8e7d6b8 is 50, key is test_row_1/B:col10/1731731166744/Put/seqid=0 2024-11-16T04:26:08,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742285_1461 (size=9757) 2024-11-16T04:26:08,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731228283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731228283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731228304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731228304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T04:26:08,558 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/1900d42858fe47718962bff7d8e7d6b8 2024-11-16T04:26:08,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/ce7419cbacf34b51ad335829d39cde1b is 50, key is test_row_1/C:col10/1731731166744/Put/seqid=0 2024-11-16T04:26:08,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742286_1462 (size=9757) 2024-11-16T04:26:08,587 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/ce7419cbacf34b51ad335829d39cde1b 2024-11-16T04:26:08,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/dbb9cff47b3e4686b698c3433947f7f8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/dbb9cff47b3e4686b698c3433947f7f8 2024-11-16T04:26:08,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731228602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731228607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/dbb9cff47b3e4686b698c3433947f7f8, entries=100, sequenceid=167, filesize=9.5 K 2024-11-16T04:26:08,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/1900d42858fe47718962bff7d8e7d6b8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1900d42858fe47718962bff7d8e7d6b8 2024-11-16T04:26:08,613 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1900d42858fe47718962bff7d8e7d6b8, entries=100, sequenceid=167, filesize=9.5 K 2024-11-16T04:26:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/ce7419cbacf34b51ad335829d39cde1b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ce7419cbacf34b51ad335829d39cde1b 2024-11-16T04:26:08,617 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ce7419cbacf34b51ad335829d39cde1b, entries=100, sequenceid=167, filesize=9.5 K 2024-11-16T04:26:08,618 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 3c5eac5e1b52fdf0c230183879dedd11 in 917ms, sequenceid=167, compaction requested=false 2024-11-16T04:26:08,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:08,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:08,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-16T04:26:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-16T04:26:08,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T04:26:08,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-11-16T04:26:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:08,621 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:08,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-16T04:26:08,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3880 sec 2024-11-16T04:26:08,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d91248b9e0524348946333844bdaaf1e is 50, key is test_row_0/A:col10/1731731168619/Put/seqid=0 2024-11-16T04:26:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731228632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742287_1463 (size=16931) 2024-11-16T04:26:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731228633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d91248b9e0524348946333844bdaaf1e 2024-11-16T04:26:08,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/72ab90a4174c4e95a0be62474136a828 is 50, key is test_row_0/B:col10/1731731168619/Put/seqid=0 2024-11-16T04:26:08,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742288_1464 (size=12151) 2024-11-16T04:26:08,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/72ab90a4174c4e95a0be62474136a828 2024-11-16T04:26:08,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/22d71c72f87243538b70df7766ca7923 is 50, key is test_row_0/C:col10/1731731168619/Put/seqid=0 2024-11-16T04:26:08,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742289_1465 (size=12151) 2024-11-16T04:26:08,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731228743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731228743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731228953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:08,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731228963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/22d71c72f87243538b70df7766ca7923 2024-11-16T04:26:09,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d91248b9e0524348946333844bdaaf1e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d91248b9e0524348946333844bdaaf1e 2024-11-16T04:26:09,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d91248b9e0524348946333844bdaaf1e, entries=250, sequenceid=198, filesize=16.5 K 2024-11-16T04:26:09,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/72ab90a4174c4e95a0be62474136a828 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/72ab90a4174c4e95a0be62474136a828 2024-11-16T04:26:09,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/72ab90a4174c4e95a0be62474136a828, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:26:09,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/22d71c72f87243538b70df7766ca7923 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/22d71c72f87243538b70df7766ca7923 2024-11-16T04:26:09,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/22d71c72f87243538b70df7766ca7923, entries=150, sequenceid=198, filesize=11.9 K 2024-11-16T04:26:09,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 3c5eac5e1b52fdf0c230183879dedd11 in 473ms, sequenceid=198, compaction requested=true 2024-11-16T04:26:09,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:09,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:09,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:09,094 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:09,094 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:09,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:09,095 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:09,095 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:09,095 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:09,095 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:09,095 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,095 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,095 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8757205544464cc5a582ef9744674701, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1900d42858fe47718962bff7d8e7d6b8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/72ab90a4174c4e95a0be62474136a828] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=33.6 K 2024-11-16T04:26:09,095 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/7aecb901fd604d2e9f97dd7bcda00bcf, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/dbb9cff47b3e4686b698c3433947f7f8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d91248b9e0524348946333844bdaaf1e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=38.3 K 2024-11-16T04:26:09,095 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8757205544464cc5a582ef9744674701, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731731165558 2024-11-16T04:26:09,095 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aecb901fd604d2e9f97dd7bcda00bcf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731731165558 2024-11-16T04:26:09,096 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1900d42858fe47718962bff7d8e7d6b8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731731166741 2024-11-16T04:26:09,096 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbb9cff47b3e4686b698c3433947f7f8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731731166741 2024-11-16T04:26:09,096 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 72ab90a4174c4e95a0be62474136a828, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731167951 2024-11-16T04:26:09,096 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91248b9e0524348946333844bdaaf1e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731167917 2024-11-16T04:26:09,103 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:09,103 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/0c5ba5193ed44da79d8acdac85deeb05 is 50, key is test_row_0/B:col10/1731731168619/Put/seqid=0 2024-11-16T04:26:09,106 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:09,106 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/c219aa7bcc1246f0805acc0c27bb6f49 is 50, key is test_row_0/A:col10/1731731168619/Put/seqid=0 2024-11-16T04:26:09,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742291_1467 (size=12595) 2024-11-16T04:26:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:09,136 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:09,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:26:09,138 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/c219aa7bcc1246f0805acc0c27bb6f49 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c219aa7bcc1246f0805acc0c27bb6f49 2024-11-16T04:26:09,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742290_1466 (size=12595) 2024-11-16T04:26:09,146 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into c219aa7bcc1246f0805acc0c27bb6f49(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:09,146 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:09,146 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731169094; duration=0sec 2024-11-16T04:26:09,146 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:09,146 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:09,146 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:09,148 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:09,148 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:09,148 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,149 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/0c5ba5193ed44da79d8acdac85deeb05 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/0c5ba5193ed44da79d8acdac85deeb05 2024-11-16T04:26:09,149 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/6be8c97fc4f34c1993f08b5fba06dab1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ce7419cbacf34b51ad335829d39cde1b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/22d71c72f87243538b70df7766ca7923] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=33.6 K 2024-11-16T04:26:09,149 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be8c97fc4f34c1993f08b5fba06dab1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731731165558 2024-11-16T04:26:09,150 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce7419cbacf34b51ad335829d39cde1b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731731166741 2024-11-16T04:26:09,150 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d71c72f87243538b70df7766ca7923, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731167951 2024-11-16T04:26:09,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/2c271e7ab11d44d9b6c1e6425dbf4731 is 50, key is test_row_0/A:col10/1731731168625/Put/seqid=0 2024-11-16T04:26:09,155 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 0c5ba5193ed44da79d8acdac85deeb05(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:09,155 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:09,155 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731169094; duration=0sec 2024-11-16T04:26:09,155 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:09,156 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:09,158 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:09,158 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/b3b71424a60140aa86aac058a21a884c is 50, key is test_row_0/C:col10/1731731168619/Put/seqid=0 2024-11-16T04:26:09,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742292_1468 (size=14541) 2024-11-16T04:26:09,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/2c271e7ab11d44d9b6c1e6425dbf4731 2024-11-16T04:26:09,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742293_1469 (size=12595) 2024-11-16T04:26:09,174 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/b3b71424a60140aa86aac058a21a884c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b3b71424a60140aa86aac058a21a884c 2024-11-16T04:26:09,179 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into b3b71424a60140aa86aac058a21a884c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:09,179 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:09,179 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731169095; duration=0sec 2024-11-16T04:26:09,179 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:09,179 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:09,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/f9ce7ce6ee16464c8e8305c65db5b0a6 is 50, key is test_row_0/B:col10/1731731168625/Put/seqid=0 2024-11-16T04:26:09,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742294_1470 (size=12151) 2024-11-16T04:26:09,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731229229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731229230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731229263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731229273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731229343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731229343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-16T04:26:09,384 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:09,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-16T04:26:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T04:26:09,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:09,386 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:09,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T04:26:09,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T04:26:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:09,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731229563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731229563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/f9ce7ce6ee16464c8e8305c65db5b0a6 2024-11-16T04:26:09,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/394918ecab3c450eb7c8a89033e6ac56 is 50, key is test_row_0/C:col10/1731731168625/Put/seqid=0 2024-11-16T04:26:09,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742295_1471 (size=12151) 2024-11-16T04:26:09,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T04:26:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T04:26:09,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731229774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731229783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T04:26:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731229874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731229874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:09,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:09,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T04:26:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:09,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:09,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:10,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/394918ecab3c450eb7c8a89033e6ac56 2024-11-16T04:26:10,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/2c271e7ab11d44d9b6c1e6425dbf4731 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/2c271e7ab11d44d9b6c1e6425dbf4731 2024-11-16T04:26:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T04:26:10,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/2c271e7ab11d44d9b6c1e6425dbf4731, entries=200, sequenceid=209, filesize=14.2 K 2024-11-16T04:26:10,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/f9ce7ce6ee16464c8e8305c65db5b0a6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/f9ce7ce6ee16464c8e8305c65db5b0a6 2024-11-16T04:26:10,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/f9ce7ce6ee16464c8e8305c65db5b0a6, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T04:26:10,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/394918ecab3c450eb7c8a89033e6ac56 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/394918ecab3c450eb7c8a89033e6ac56 2024-11-16T04:26:10,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/394918ecab3c450eb7c8a89033e6ac56, entries=150, sequenceid=209, filesize=11.9 K 2024-11-16T04:26:10,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3c5eac5e1b52fdf0c230183879dedd11 in 888ms, sequenceid=209, compaction requested=false 2024-11-16T04:26:10,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:10,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-16T04:26:10,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:10,151 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:26:10,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/68f87c553a83407abaea721756487d2d is 50, key is test_row_0/A:col10/1731731169226/Put/seqid=0 2024-11-16T04:26:10,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742296_1472 (size=12151) 2024-11-16T04:26:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:10,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731230414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731230421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T04:26:10,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731230533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731230533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,564 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/68f87c553a83407abaea721756487d2d 2024-11-16T04:26:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/c6914644b7be444391d85bffda9c6f63 is 50, key is test_row_0/B:col10/1731731169226/Put/seqid=0 2024-11-16T04:26:10,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742297_1473 (size=12151) 2024-11-16T04:26:10,598 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/c6914644b7be444391d85bffda9c6f63 2024-11-16T04:26:10,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/01813ac495ec49bf8d4ae634cb6ecd0f is 50, key is test_row_0/C:col10/1731731169226/Put/seqid=0 2024-11-16T04:26:10,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742298_1474 (size=12151) 2024-11-16T04:26:10,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731230743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731230743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731230793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731230794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:10,796 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:10,796 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,020 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/01813ac495ec49bf8d4ae634cb6ecd0f 2024-11-16T04:26:11,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/68f87c553a83407abaea721756487d2d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/68f87c553a83407abaea721756487d2d 2024-11-16T04:26:11,025 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/68f87c553a83407abaea721756487d2d, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T04:26:11,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/c6914644b7be444391d85bffda9c6f63 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c6914644b7be444391d85bffda9c6f63 2024-11-16T04:26:11,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c6914644b7be444391d85bffda9c6f63, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T04:26:11,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/01813ac495ec49bf8d4ae634cb6ecd0f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/01813ac495ec49bf8d4ae634cb6ecd0f 2024-11-16T04:26:11,032 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/01813ac495ec49bf8d4ae634cb6ecd0f, entries=150, sequenceid=237, filesize=11.9 K 2024-11-16T04:26:11,032 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3c5eac5e1b52fdf0c230183879dedd11 in 881ms, sequenceid=237, compaction requested=true 2024-11-16T04:26:11,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:11,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:11,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-16T04:26:11,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-16T04:26:11,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-16T04:26:11,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-11-16T04:26:11,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6520 sec 2024-11-16T04:26:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:11,069 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:11,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:26:11,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/8343381f9e1444e5baa793a1f6f152db is 50, key is test_row_0/A:col10/1731731171068/Put/seqid=0 2024-11-16T04:26:11,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742299_1475 (size=16931) 2024-11-16T04:26:11,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/8343381f9e1444e5baa793a1f6f152db 2024-11-16T04:26:11,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/dd44bdef461f4958b1bbaa52dd4e8d86 is 50, key is test_row_0/B:col10/1731731171068/Put/seqid=0 2024-11-16T04:26:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742300_1476 (size=12151) 2024-11-16T04:26:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731231164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731231169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731231274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731231283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731231483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/dd44bdef461f4958b1bbaa52dd4e8d86 2024-11-16T04:26:11,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731231504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/fbb62ec4633d4845aab98c7f5f74a68c is 50, key is test_row_0/C:col10/1731731171068/Put/seqid=0 2024-11-16T04:26:11,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742301_1477 (size=12151) 2024-11-16T04:26:11,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/fbb62ec4633d4845aab98c7f5f74a68c 2024-11-16T04:26:11,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/8343381f9e1444e5baa793a1f6f152db as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8343381f9e1444e5baa793a1f6f152db 2024-11-16T04:26:11,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8343381f9e1444e5baa793a1f6f152db, entries=250, sequenceid=250, filesize=16.5 K 2024-11-16T04:26:11,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/dd44bdef461f4958b1bbaa52dd4e8d86 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/dd44bdef461f4958b1bbaa52dd4e8d86 2024-11-16T04:26:11,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/dd44bdef461f4958b1bbaa52dd4e8d86, entries=150, sequenceid=250, filesize=11.9 K 2024-11-16T04:26:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/fbb62ec4633d4845aab98c7f5f74a68c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/fbb62ec4633d4845aab98c7f5f74a68c 2024-11-16T04:26:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T04:26:11,533 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:11,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-16T04:26:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T04:26:11,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:11,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:11,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:11,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/fbb62ec4633d4845aab98c7f5f74a68c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-16T04:26:11,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3c5eac5e1b52fdf0c230183879dedd11 in 476ms, sequenceid=250, compaction requested=true 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:11,545 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:11,545 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:11,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:11,546 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:11,546 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:11,546 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:11,546 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:11,547 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:11,547 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:11,547 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c219aa7bcc1246f0805acc0c27bb6f49, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/2c271e7ab11d44d9b6c1e6425dbf4731, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/68f87c553a83407abaea721756487d2d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8343381f9e1444e5baa793a1f6f152db] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=54.9 K 2024-11-16T04:26:11,547 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/0c5ba5193ed44da79d8acdac85deeb05, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/f9ce7ce6ee16464c8e8305c65db5b0a6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c6914644b7be444391d85bffda9c6f63, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/dd44bdef461f4958b1bbaa52dd4e8d86] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=47.9 K 2024-11-16T04:26:11,547 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c219aa7bcc1246f0805acc0c27bb6f49, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731167951 2024-11-16T04:26:11,547 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c5ba5193ed44da79d8acdac85deeb05, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731167951 2024-11-16T04:26:11,547 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c271e7ab11d44d9b6c1e6425dbf4731, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731731168625 2024-11-16T04:26:11,547 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ce7ce6ee16464c8e8305c65db5b0a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731731168625 2024-11-16T04:26:11,547 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68f87c553a83407abaea721756487d2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731731169226 2024-11-16T04:26:11,547 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting c6914644b7be444391d85bffda9c6f63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731731169226 2024-11-16T04:26:11,548 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8343381f9e1444e5baa793a1f6f152db, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731731170400 2024-11-16T04:26:11,548 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting dd44bdef461f4958b1bbaa52dd4e8d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731731170402 2024-11-16T04:26:11,554 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:11,554 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/182f7372b65445f3b2d568975f5b56c1 is 50, key is test_row_0/B:col10/1731731171068/Put/seqid=0 2024-11-16T04:26:11,555 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:11,555 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/bf9cf9b90c7746a9960c78144bcad24d is 50, key is test_row_0/A:col10/1731731171068/Put/seqid=0 2024-11-16T04:26:11,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742302_1478 (size=12731) 2024-11-16T04:26:11,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742303_1479 (size=12731) 2024-11-16T04:26:11,568 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/bf9cf9b90c7746a9960c78144bcad24d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bf9cf9b90c7746a9960c78144bcad24d 2024-11-16T04:26:11,572 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into bf9cf9b90c7746a9960c78144bcad24d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:11,572 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:11,572 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=12, startTime=1731731171545; duration=0sec 2024-11-16T04:26:11,572 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:11,572 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:11,572 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:11,573 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:11,573 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:11,573 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:11,573 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b3b71424a60140aa86aac058a21a884c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/394918ecab3c450eb7c8a89033e6ac56, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/01813ac495ec49bf8d4ae634cb6ecd0f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/fbb62ec4633d4845aab98c7f5f74a68c] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=47.9 K 2024-11-16T04:26:11,573 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3b71424a60140aa86aac058a21a884c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731731167951 2024-11-16T04:26:11,573 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 394918ecab3c450eb7c8a89033e6ac56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731731168625 2024-11-16T04:26:11,573 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01813ac495ec49bf8d4ae634cb6ecd0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731731169226 2024-11-16T04:26:11,574 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbb62ec4633d4845aab98c7f5f74a68c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731731170402 2024-11-16T04:26:11,580 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:11,581 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/0b5140be63ea4c98a4b844cfd6ebd9ce is 50, key is test_row_0/C:col10/1731731171068/Put/seqid=0 2024-11-16T04:26:11,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742304_1480 (size=12731) 2024-11-16T04:26:11,593 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/0b5140be63ea4c98a4b844cfd6ebd9ce as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/0b5140be63ea4c98a4b844cfd6ebd9ce 2024-11-16T04:26:11,597 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 0b5140be63ea4c98a4b844cfd6ebd9ce(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:11,597 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:11,597 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=12, startTime=1731731171545; duration=0sec 2024-11-16T04:26:11,597 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:11,597 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T04:26:11,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-16T04:26:11,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:11,689 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:26:11,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/5e688391903045e490674a49b4108e71 is 50, key is test_row_0/A:col10/1731731171156/Put/seqid=0 2024-11-16T04:26:11,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742305_1481 (size=12301) 2024-11-16T04:26:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:11,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:11,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731231841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T04:26:11,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731231847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,963 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/182f7372b65445f3b2d568975f5b56c1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/182f7372b65445f3b2d568975f5b56c1 2024-11-16T04:26:11,966 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 182f7372b65445f3b2d568975f5b56c1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:11,966 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:11,966 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=12, startTime=1731731171545; duration=0sec 2024-11-16T04:26:11,967 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:11,967 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:11,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731231963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731231963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:11,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,096 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/5e688391903045e490674a49b4108e71 2024-11-16T04:26:12,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/708fc586bdf2470da2e2185168dad108 is 50, key is test_row_0/B:col10/1731731171156/Put/seqid=0 2024-11-16T04:26:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742306_1482 (size=12301) 2024-11-16T04:26:12,127 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/708fc586bdf2470da2e2185168dad108 2024-11-16T04:26:12,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/c45d32f8deed4a279493f8b8feadbf7c is 50, key is test_row_0/C:col10/1731731171156/Put/seqid=0 2024-11-16T04:26:12,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742307_1483 (size=12301) 2024-11-16T04:26:12,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/c45d32f8deed4a279493f8b8feadbf7c 2024-11-16T04:26:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T04:26:12,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/5e688391903045e490674a49b4108e71 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/5e688391903045e490674a49b4108e71 2024-11-16T04:26:12,168 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/5e688391903045e490674a49b4108e71, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T04:26:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/708fc586bdf2470da2e2185168dad108 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/708fc586bdf2470da2e2185168dad108 2024-11-16T04:26:12,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/708fc586bdf2470da2e2185168dad108, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T04:26:12,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/c45d32f8deed4a279493f8b8feadbf7c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/c45d32f8deed4a279493f8b8feadbf7c 2024-11-16T04:26:12,177 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/c45d32f8deed4a279493f8b8feadbf7c, entries=150, sequenceid=275, filesize=12.0 K 2024-11-16T04:26:12,178 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3c5eac5e1b52fdf0c230183879dedd11 in 489ms, sequenceid=275, compaction requested=false 2024-11-16T04:26:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:12,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-16T04:26:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-16T04:26:12,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:12,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T04:26:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:12,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T04:26:12,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 645 msec 2024-11-16T04:26:12,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 650 msec 2024-11-16T04:26:12,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/10fdc0d3a0f24686a9b1441345982d81 is 50, key is test_row_0/A:col10/1731731171845/Put/seqid=0 2024-11-16T04:26:12,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742308_1484 (size=14741) 2024-11-16T04:26:12,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731232260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731232262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731232373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731232384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731232583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731232600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/10fdc0d3a0f24686a9b1441345982d81 2024-11-16T04:26:12,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/749a4880800941e3b7ce04038cdc2cfd is 50, key is test_row_0/B:col10/1731731171845/Put/seqid=0 2024-11-16T04:26:12,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742309_1485 (size=12301) 2024-11-16T04:26:12,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/749a4880800941e3b7ce04038cdc2cfd 2024-11-16T04:26:12,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/82a61b2e3e40426f95d413ee275512f2 is 50, key is test_row_0/C:col10/1731731171845/Put/seqid=0 2024-11-16T04:26:12,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742310_1486 (size=12301) 2024-11-16T04:26:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-16T04:26:12,673 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:12,674 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T04:26:12,675 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T04:26:12,675 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:12,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T04:26:12,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731232813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,821 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T04:26:12,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:12,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:12,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:12,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:12,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:12,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731232824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,831 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731232903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:12,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731232914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:12,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T04:26:12,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:12,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:12,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:12,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:12,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T04:26:13,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/82a61b2e3e40426f95d413ee275512f2 2024-11-16T04:26:13,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/10fdc0d3a0f24686a9b1441345982d81 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/10fdc0d3a0f24686a9b1441345982d81 2024-11-16T04:26:13,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/10fdc0d3a0f24686a9b1441345982d81, entries=200, sequenceid=290, filesize=14.4 K 2024-11-16T04:26:13,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/749a4880800941e3b7ce04038cdc2cfd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/749a4880800941e3b7ce04038cdc2cfd 2024-11-16T04:26:13,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/749a4880800941e3b7ce04038cdc2cfd, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T04:26:13,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/82a61b2e3e40426f95d413ee275512f2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/82a61b2e3e40426f95d413ee275512f2 2024-11-16T04:26:13,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/82a61b2e3e40426f95d413ee275512f2, entries=150, sequenceid=290, filesize=12.0 K 2024-11-16T04:26:13,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c5eac5e1b52fdf0c230183879dedd11 in 899ms, sequenceid=290, compaction requested=true 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:13,081 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:13,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T04:26:13,082 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:13,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:13,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:13,082 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:13,082 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bf9cf9b90c7746a9960c78144bcad24d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/5e688391903045e490674a49b4108e71, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/10fdc0d3a0f24686a9b1441345982d81] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=38.8 K 2024-11-16T04:26:13,082 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9cf9b90c7746a9960c78144bcad24d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731731170402 2024-11-16T04:26:13,084 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:13,084 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e688391903045e490674a49b4108e71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731731171153 2024-11-16T04:26:13,084 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:13,084 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:13,085 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/182f7372b65445f3b2d568975f5b56c1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/708fc586bdf2470da2e2185168dad108, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/749a4880800941e3b7ce04038cdc2cfd] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=36.5 K 2024-11-16T04:26:13,085 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10fdc0d3a0f24686a9b1441345982d81, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731731171812 2024-11-16T04:26:13,085 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 182f7372b65445f3b2d568975f5b56c1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731731170402 2024-11-16T04:26:13,085 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 708fc586bdf2470da2e2185168dad108, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731731171153 2024-11-16T04:26:13,085 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 749a4880800941e3b7ce04038cdc2cfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731731171812 2024-11-16T04:26:13,093 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:13,094 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/be5410f233424cad9928cec895389ec5 is 50, key is test_row_0/B:col10/1731731171845/Put/seqid=0 2024-11-16T04:26:13,095 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:13,095 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc is 50, key is test_row_0/A:col10/1731731171845/Put/seqid=0 2024-11-16T04:26:13,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742311_1487 (size=12983) 2024-11-16T04:26:13,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742312_1488 (size=12983) 2024-11-16T04:26:13,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T04:26:13,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:13,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T04:26:13,137 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc 2024-11-16T04:26:13,141 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into 3a99c9fa0f2e4a95b50ff7ade3ef2fbc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:13,141 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:13,142 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731173081; duration=0sec 2024-11-16T04:26:13,142 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:13,142 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:13,142 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:13,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d8387a0e3ffa40598c5e9a4f4804606d is 50, key is test_row_0/A:col10/1731731172260/Put/seqid=0 2024-11-16T04:26:13,143 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:13,143 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:13,143 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:13,143 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/0b5140be63ea4c98a4b844cfd6ebd9ce, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/c45d32f8deed4a279493f8b8feadbf7c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/82a61b2e3e40426f95d413ee275512f2] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=36.5 K 2024-11-16T04:26:13,144 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b5140be63ea4c98a4b844cfd6ebd9ce, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731731170402 2024-11-16T04:26:13,144 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c45d32f8deed4a279493f8b8feadbf7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731731171153 2024-11-16T04:26:13,144 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a61b2e3e40426f95d413ee275512f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731731171812 2024-11-16T04:26:13,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742313_1489 (size=12301) 2024-11-16T04:26:13,150 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#417 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:13,150 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/997536ac45eb48c59a99e2e9bc653c01 is 50, key is test_row_0/C:col10/1731731171845/Put/seqid=0 2024-11-16T04:26:13,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d8387a0e3ffa40598c5e9a4f4804606d 2024-11-16T04:26:13,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742314_1490 (size=12983) 2024-11-16T04:26:13,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/20b92d2cb8e642d299e373658a5c970d is 50, key is test_row_0/B:col10/1731731172260/Put/seqid=0 2024-11-16T04:26:13,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742315_1491 (size=12301) 2024-11-16T04:26:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T04:26:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:13,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:13,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731233459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:13,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731233460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:13,509 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/be5410f233424cad9928cec895389ec5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/be5410f233424cad9928cec895389ec5 2024-11-16T04:26:13,512 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into be5410f233424cad9928cec895389ec5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:13,512 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:13,512 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731173081; duration=0sec 2024-11-16T04:26:13,512 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:13,512 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731233573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:13,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731233573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:13,578 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/997536ac45eb48c59a99e2e9bc653c01 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/997536ac45eb48c59a99e2e9bc653c01 2024-11-16T04:26:13,583 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 997536ac45eb48c59a99e2e9bc653c01(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:13,583 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:13,583 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731173081; duration=0sec 2024-11-16T04:26:13,583 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:13,583 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:13,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/20b92d2cb8e642d299e373658a5c970d 2024-11-16T04:26:13,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/419b5f408df84090be49acb34a27db3e is 50, key is test_row_0/C:col10/1731731172260/Put/seqid=0 2024-11-16T04:26:13,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742316_1492 (size=12301) 2024-11-16T04:26:13,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731233784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:13,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731233784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:13,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T04:26:13,997 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/419b5f408df84090be49acb34a27db3e 2024-11-16T04:26:14,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d8387a0e3ffa40598c5e9a4f4804606d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d8387a0e3ffa40598c5e9a4f4804606d 2024-11-16T04:26:14,008 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d8387a0e3ffa40598c5e9a4f4804606d, entries=150, sequenceid=312, filesize=12.0 K 2024-11-16T04:26:14,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/20b92d2cb8e642d299e373658a5c970d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/20b92d2cb8e642d299e373658a5c970d 2024-11-16T04:26:14,015 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/20b92d2cb8e642d299e373658a5c970d, entries=150, sequenceid=312, filesize=12.0 K 2024-11-16T04:26:14,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/419b5f408df84090be49acb34a27db3e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/419b5f408df84090be49acb34a27db3e 2024-11-16T04:26:14,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/419b5f408df84090be49acb34a27db3e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-16T04:26:14,019 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3c5eac5e1b52fdf0c230183879dedd11 in 883ms, sequenceid=312, compaction requested=false 2024-11-16T04:26:14,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:14,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:14,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T04:26:14,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T04:26:14,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T04:26:14,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-16T04:26:14,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.3480 sec 2024-11-16T04:26:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:14,096 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:14,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:26:14,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/85852223c36c490bb0165a6808cb7b96 is 50, key is test_row_0/A:col10/1731731173458/Put/seqid=0 2024-11-16T04:26:14,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742317_1493 (size=14741) 2024-11-16T04:26:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731234167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731234173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731234284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731234284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/85852223c36c490bb0165a6808cb7b96 2024-11-16T04:26:14,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731234503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731234503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/14c8de261de94dd589951be5834f54e8 is 50, key is test_row_0/B:col10/1731731173458/Put/seqid=0 2024-11-16T04:26:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742318_1494 (size=12301) 2024-11-16T04:26:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T04:26:14,824 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:14,824 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T04:26:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T04:26:14,826 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:14,827 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:14,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:14,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731234824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731234824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:14,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/14c8de261de94dd589951be5834f54e8 2024-11-16T04:26:14,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/dbac8e99c30d491ca069bce8267c91fb is 50, key is test_row_0/C:col10/1731731173458/Put/seqid=0 2024-11-16T04:26:14,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742319_1495 (size=12301) 2024-11-16T04:26:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T04:26:14,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:14,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:14,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:14,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:14,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:14,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T04:26:15,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:15,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:15,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/dbac8e99c30d491ca069bce8267c91fb 2024-11-16T04:26:15,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/85852223c36c490bb0165a6808cb7b96 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/85852223c36c490bb0165a6808cb7b96 2024-11-16T04:26:15,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/85852223c36c490bb0165a6808cb7b96, entries=200, sequenceid=330, filesize=14.4 K 2024-11-16T04:26:15,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/14c8de261de94dd589951be5834f54e8 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/14c8de261de94dd589951be5834f54e8 2024-11-16T04:26:15,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/14c8de261de94dd589951be5834f54e8, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T04:26:15,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/dbac8e99c30d491ca069bce8267c91fb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dbac8e99c30d491ca069bce8267c91fb 2024-11-16T04:26:15,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dbac8e99c30d491ca069bce8267c91fb, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T04:26:15,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3c5eac5e1b52fdf0c230183879dedd11 in 1247ms, sequenceid=330, compaction requested=true 2024-11-16T04:26:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:15,344 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:15,344 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:15,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:15,344 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:15,344 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:15,344 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:15,344 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:15,344 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,344 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,345 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/be5410f233424cad9928cec895389ec5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/20b92d2cb8e642d299e373658a5c970d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/14c8de261de94dd589951be5834f54e8] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=36.7 K 2024-11-16T04:26:15,345 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d8387a0e3ffa40598c5e9a4f4804606d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/85852223c36c490bb0165a6808cb7b96] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=39.1 K 2024-11-16T04:26:15,345 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a99c9fa0f2e4a95b50ff7ade3ef2fbc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731731171812 2024-11-16T04:26:15,345 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting be5410f233424cad9928cec895389ec5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731731171812 2024-11-16T04:26:15,345 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b92d2cb8e642d299e373658a5c970d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731731172248 2024-11-16T04:26:15,345 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8387a0e3ffa40598c5e9a4f4804606d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731731172248 2024-11-16T04:26:15,345 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c8de261de94dd589951be5834f54e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731731173458 2024-11-16T04:26:15,345 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85852223c36c490bb0165a6808cb7b96, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731731173451 2024-11-16T04:26:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:15,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:15,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-16T04:26:15,351 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:15,352 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/e7f70d023e5346539182e34ef636b5e6 is 50, key is test_row_0/A:col10/1731731173458/Put/seqid=0 2024-11-16T04:26:15,357 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:15,358 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/2ffc149d33fd41f18d53f93089d778f1 is 50, key is test_row_0/B:col10/1731731173458/Put/seqid=0 2024-11-16T04:26:15,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/643baceb0ccf4c88b202138db8df9b9c is 50, key is test_row_0/A:col10/1731731174151/Put/seqid=0 2024-11-16T04:26:15,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742320_1496 (size=13085) 2024-11-16T04:26:15,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742322_1498 (size=14741) 2024-11-16T04:26:15,375 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/e7f70d023e5346539182e34ef636b5e6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e7f70d023e5346539182e34ef636b5e6 2024-11-16T04:26:15,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/643baceb0ccf4c88b202138db8df9b9c 2024-11-16T04:26:15,379 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into e7f70d023e5346539182e34ef636b5e6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:15,379 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:15,379 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731175343; duration=0sec 2024-11-16T04:26:15,380 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:15,380 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:15,380 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:15,385 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:15,385 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:15,385 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,385 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/997536ac45eb48c59a99e2e9bc653c01, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/419b5f408df84090be49acb34a27db3e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dbac8e99c30d491ca069bce8267c91fb] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=36.7 K 2024-11-16T04:26:15,386 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 997536ac45eb48c59a99e2e9bc653c01, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731731171812 2024-11-16T04:26:15,386 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 419b5f408df84090be49acb34a27db3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731731172248 2024-11-16T04:26:15,386 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbac8e99c30d491ca069bce8267c91fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731731173458 2024-11-16T04:26:15,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742321_1497 (size=13085) 2024-11-16T04:26:15,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8c0c07a3bc4847afb4d60e8b3759f9e7 is 50, key is test_row_0/B:col10/1731731174151/Put/seqid=0 2024-11-16T04:26:15,397 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:15,397 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d032be82417a4b46910eda3c29e466af is 50, key is test_row_0/C:col10/1731731173458/Put/seqid=0 2024-11-16T04:26:15,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742324_1500 (size=13085) 2024-11-16T04:26:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731235401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731235402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:15,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742323_1499 (size=12301) 2024-11-16T04:26:15,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T04:26:15,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731235513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:15,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731235513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:15,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:15,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:15,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731235724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731235724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:15,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:15,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:15,794 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/2ffc149d33fd41f18d53f93089d778f1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/2ffc149d33fd41f18d53f93089d778f1 2024-11-16T04:26:15,798 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 2ffc149d33fd41f18d53f93089d778f1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:15,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:15,798 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731175344; duration=0sec 2024-11-16T04:26:15,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:15,798 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:15,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8c0c07a3bc4847afb4d60e8b3759f9e7 2024-11-16T04:26:15,813 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/d032be82417a4b46910eda3c29e466af as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d032be82417a4b46910eda3c29e466af 2024-11-16T04:26:15,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/60ec2f5a638c4218babc5e1db4ba8afe is 50, key is test_row_0/C:col10/1731731174151/Put/seqid=0 2024-11-16T04:26:15,824 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into d032be82417a4b46910eda3c29e466af(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:15,824 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:15,824 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731175344; duration=0sec 2024-11-16T04:26:15,824 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:15,824 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:15,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742325_1501 (size=12301) 2024-11-16T04:26:15,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/60ec2f5a638c4218babc5e1db4ba8afe 2024-11-16T04:26:15,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/643baceb0ccf4c88b202138db8df9b9c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/643baceb0ccf4c88b202138db8df9b9c 2024-11-16T04:26:15,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/643baceb0ccf4c88b202138db8df9b9c, entries=200, sequenceid=352, filesize=14.4 K 2024-11-16T04:26:15,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8c0c07a3bc4847afb4d60e8b3759f9e7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8c0c07a3bc4847afb4d60e8b3759f9e7 2024-11-16T04:26:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8c0c07a3bc4847afb4d60e8b3759f9e7, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T04:26:15,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/60ec2f5a638c4218babc5e1db4ba8afe as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/60ec2f5a638c4218babc5e1db4ba8afe 2024-11-16T04:26:15,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/60ec2f5a638c4218babc5e1db4ba8afe, entries=150, sequenceid=352, filesize=12.0 K 2024-11-16T04:26:15,856 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3c5eac5e1b52fdf0c230183879dedd11 in 505ms, sequenceid=352, compaction requested=false 2024-11-16T04:26:15,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:15,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T04:26:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:15,897 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-16T04:26:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6817df2ec0fd4b599db03c8ea954f247 is 50, key is test_row_0/A:col10/1731731175385/Put/seqid=0 2024-11-16T04:26:15,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742326_1502 (size=12301) 2024-11-16T04:26:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T04:26:16,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:16,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:16,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731236110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731236112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731236223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731236223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6817df2ec0fd4b599db03c8ea954f247 2024-11-16T04:26:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/a5fc4ef05e3e4d7b822f81feadb49349 is 50, key is test_row_0/B:col10/1731731175385/Put/seqid=0 2024-11-16T04:26:16,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742327_1503 (size=12301) 2024-11-16T04:26:16,315 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/a5fc4ef05e3e4d7b822f81feadb49349 2024-11-16T04:26:16,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/8b79a4891f074e1aab3e939524d11a6a is 50, key is test_row_0/C:col10/1731731175385/Put/seqid=0 2024-11-16T04:26:16,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742328_1504 (size=12301) 2024-11-16T04:26:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731236433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731236434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,727 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/8b79a4891f074e1aab3e939524d11a6a 2024-11-16T04:26:16,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6817df2ec0fd4b599db03c8ea954f247 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6817df2ec0fd4b599db03c8ea954f247 2024-11-16T04:26:16,733 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6817df2ec0fd4b599db03c8ea954f247, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T04:26:16,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/a5fc4ef05e3e4d7b822f81feadb49349 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a5fc4ef05e3e4d7b822f81feadb49349 2024-11-16T04:26:16,736 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a5fc4ef05e3e4d7b822f81feadb49349, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T04:26:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/8b79a4891f074e1aab3e939524d11a6a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8b79a4891f074e1aab3e939524d11a6a 2024-11-16T04:26:16,747 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8b79a4891f074e1aab3e939524d11a6a, entries=150, sequenceid=369, filesize=12.0 K 2024-11-16T04:26:16,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3c5eac5e1b52fdf0c230183879dedd11 in 852ms, sequenceid=369, compaction requested=true 2024-11-16T04:26:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:16,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T04:26:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T04:26:16,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T04:26:16,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9240 sec 2024-11-16T04:26:16,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9300 sec 2024-11-16T04:26:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:16,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:16,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:26:16,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/c4b6bd7de405482eb0d7411181f64524 is 50, key is test_row_0/A:col10/1731731176110/Put/seqid=0 2024-11-16T04:26:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742329_1505 (size=17181) 2024-11-16T04:26:16,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731236812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731236819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58578 deadline: 1731731236844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,855 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58580 deadline: 1731731236863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,874 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731236934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:16,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731236943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:16,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T04:26:16,974 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:16,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T04:26:16,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T04:26:16,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:16,976 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:16,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T04:26:17,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T04:26:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731237153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731237153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:17,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/c4b6bd7de405482eb0d7411181f64524 2024-11-16T04:26:17,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/1f1025b42ea249e19ebb75bc19d298ea is 50, key is test_row_0/B:col10/1731731176110/Put/seqid=0 2024-11-16T04:26:17,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742330_1506 (size=12301) 2024-11-16T04:26:17,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/1f1025b42ea249e19ebb75bc19d298ea 2024-11-16T04:26:17,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/ef3ebe40cbe246d88a46d77973740469 is 50, key is test_row_0/C:col10/1731731176110/Put/seqid=0 2024-11-16T04:26:17,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742331_1507 (size=12301) 2024-11-16T04:26:17,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T04:26:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T04:26:17,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T04:26:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731237474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731237474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:17,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T04:26:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:17,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/ef3ebe40cbe246d88a46d77973740469 2024-11-16T04:26:17,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T04:26:17,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/c4b6bd7de405482eb0d7411181f64524 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c4b6bd7de405482eb0d7411181f64524 2024-11-16T04:26:17,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c4b6bd7de405482eb0d7411181f64524, entries=250, sequenceid=393, filesize=16.8 K 2024-11-16T04:26:17,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/1f1025b42ea249e19ebb75bc19d298ea as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1f1025b42ea249e19ebb75bc19d298ea 2024-11-16T04:26:17,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1f1025b42ea249e19ebb75bc19d298ea, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T04:26:17,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/ef3ebe40cbe246d88a46d77973740469 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ef3ebe40cbe246d88a46d77973740469 2024-11-16T04:26:17,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ef3ebe40cbe246d88a46d77973740469, entries=150, sequenceid=393, filesize=12.0 K 2024-11-16T04:26:17,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3c5eac5e1b52fdf0c230183879dedd11 in 846ms, sequenceid=393, compaction requested=true 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:17,617 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:17,617 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:17,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:17,618 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:17,618 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:17,618 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:17,618 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:17,618 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,618 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,618 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e7f70d023e5346539182e34ef636b5e6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/643baceb0ccf4c88b202138db8df9b9c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6817df2ec0fd4b599db03c8ea954f247, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c4b6bd7de405482eb0d7411181f64524] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=56.0 K 2024-11-16T04:26:17,618 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/2ffc149d33fd41f18d53f93089d778f1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8c0c07a3bc4847afb4d60e8b3759f9e7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a5fc4ef05e3e4d7b822f81feadb49349, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1f1025b42ea249e19ebb75bc19d298ea] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=48.8 K 2024-11-16T04:26:17,619 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7f70d023e5346539182e34ef636b5e6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731731173458 2024-11-16T04:26:17,619 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffc149d33fd41f18d53f93089d778f1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731731173458 2024-11-16T04:26:17,620 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643baceb0ccf4c88b202138db8df9b9c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731731174151 2024-11-16T04:26:17,620 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c0c07a3bc4847afb4d60e8b3759f9e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731731174151 2024-11-16T04:26:17,620 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a5fc4ef05e3e4d7b822f81feadb49349, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731731175351 2024-11-16T04:26:17,620 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6817df2ec0fd4b599db03c8ea954f247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731731175351 2024-11-16T04:26:17,620 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1025b42ea249e19ebb75bc19d298ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731731176083 2024-11-16T04:26:17,620 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b6bd7de405482eb0d7411181f64524, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731731176083 2024-11-16T04:26:17,629 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:17,630 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/966f6a33441447aeb034161a17a5758c is 50, key is test_row_0/B:col10/1731731176110/Put/seqid=0 2024-11-16T04:26:17,632 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:17,632 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d7d81cf5fd874700a1e879cc0f0d8417 is 50, key is test_row_0/A:col10/1731731176110/Put/seqid=0 2024-11-16T04:26:17,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742332_1508 (size=13221) 2024-11-16T04:26:17,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742333_1509 (size=13221) 2024-11-16T04:26:17,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T04:26:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:17,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:26:17,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6571177c784d40e2b99598826ae7b9fb is 50, key is test_row_0/A:col10/1731731176815/Put/seqid=0 2024-11-16T04:26:17,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742334_1510 (size=12301) 2024-11-16T04:26:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:18,063 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/d7d81cf5fd874700a1e879cc0f0d8417 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d7d81cf5fd874700a1e879cc0f0d8417 2024-11-16T04:26:18,063 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/966f6a33441447aeb034161a17a5758c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/966f6a33441447aeb034161a17a5758c 2024-11-16T04:26:18,067 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into d7d81cf5fd874700a1e879cc0f0d8417(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,067 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=12, startTime=1731731177617; duration=0sec 2024-11-16T04:26:18,067 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 966f6a33441447aeb034161a17a5758c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:18,067 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=12, startTime=1731731177617; duration=0sec 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:18,067 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:18,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T04:26:18,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:18,068 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:18,068 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d032be82417a4b46910eda3c29e466af, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/60ec2f5a638c4218babc5e1db4ba8afe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8b79a4891f074e1aab3e939524d11a6a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ef3ebe40cbe246d88a46d77973740469] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=48.8 K 2024-11-16T04:26:18,068 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d032be82417a4b46910eda3c29e466af, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731731173458 2024-11-16T04:26:18,069 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60ec2f5a638c4218babc5e1db4ba8afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731731174151 2024-11-16T04:26:18,069 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b79a4891f074e1aab3e939524d11a6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731731175351 2024-11-16T04:26:18,069 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef3ebe40cbe246d88a46d77973740469, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731731176083 2024-11-16T04:26:18,077 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:18,078 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/8035fdf363dc4c73aea80d8b1e9a9932 is 50, key is test_row_0/C:col10/1731731176110/Put/seqid=0 2024-11-16T04:26:18,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742335_1511 (size=13221) 2024-11-16T04:26:18,087 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/8035fdf363dc4c73aea80d8b1e9a9932 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8035fdf363dc4c73aea80d8b1e9a9932 2024-11-16T04:26:18,091 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 8035fdf363dc4c73aea80d8b1e9a9932(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:18,091 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,091 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=12, startTime=1731731177617; duration=0sec 2024-11-16T04:26:18,091 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:18,091 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T04:26:18,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731238121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731238125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,153 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6571177c784d40e2b99598826ae7b9fb 2024-11-16T04:26:18,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/b4afd2618424464da3df3130d62c60f2 is 50, key is test_row_0/B:col10/1731731176815/Put/seqid=0 2024-11-16T04:26:18,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742336_1512 (size=12301) 2024-11-16T04:26:18,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/b4afd2618424464da3df3130d62c60f2 2024-11-16T04:26:18,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/b0fcf220c1e14216baafc21d528ef941 is 50, key is test_row_0/C:col10/1731731176815/Put/seqid=0 2024-11-16T04:26:18,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742337_1513 (size=12301) 2024-11-16T04:26:18,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58572 deadline: 1731731238203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,207 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18348 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/b0fcf220c1e14216baafc21d528ef941 2024-11-16T04:26:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/6571177c784d40e2b99598826ae7b9fb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6571177c784d40e2b99598826ae7b9fb 2024-11-16T04:26:18,225 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6571177c784d40e2b99598826ae7b9fb, entries=150, sequenceid=405, filesize=12.0 K 2024-11-16T04:26:18,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/b4afd2618424464da3df3130d62c60f2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b4afd2618424464da3df3130d62c60f2 2024-11-16T04:26:18,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b4afd2618424464da3df3130d62c60f2, entries=150, sequenceid=405, filesize=12.0 K 2024-11-16T04:26:18,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/b0fcf220c1e14216baafc21d528ef941 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b0fcf220c1e14216baafc21d528ef941 2024-11-16T04:26:18,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731238233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b0fcf220c1e14216baafc21d528ef941, entries=150, sequenceid=405, filesize=12.0 K 2024-11-16T04:26:18,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3c5eac5e1b52fdf0c230183879dedd11 in 502ms, sequenceid=405, compaction requested=false 2024-11-16T04:26:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:18,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T04:26:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T04:26:18,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T04:26:18,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2690 sec 2024-11-16T04:26:18,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.2740 sec 2024-11-16T04:26:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:18,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:18,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:26:18,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/ec196e37be4145e5b634a84d373027be is 50, key is test_row_0/A:col10/1731731178119/Put/seqid=0 2024-11-16T04:26:18,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742338_1514 (size=14741) 2024-11-16T04:26:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731238313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731238423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731238443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731238633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/ec196e37be4145e5b634a84d373027be 2024-11-16T04:26:18,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/632c11c4abe24d89b72d38b813ae8a23 is 50, key is test_row_0/B:col10/1731731178119/Put/seqid=0 2024-11-16T04:26:18,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742339_1515 (size=12301) 2024-11-16T04:26:18,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/632c11c4abe24d89b72d38b813ae8a23 2024-11-16T04:26:18,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/3f6d4b0beef94742999b32234e2238b9 is 50, key is test_row_0/C:col10/1731731178119/Put/seqid=0 2024-11-16T04:26:18,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742340_1516 (size=12301) 2024-11-16T04:26:18,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/3f6d4b0beef94742999b32234e2238b9 2024-11-16T04:26:18,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/ec196e37be4145e5b634a84d373027be as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/ec196e37be4145e5b634a84d373027be 2024-11-16T04:26:18,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/ec196e37be4145e5b634a84d373027be, entries=200, sequenceid=433, filesize=14.4 K 2024-11-16T04:26:18,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/632c11c4abe24d89b72d38b813ae8a23 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/632c11c4abe24d89b72d38b813ae8a23 2024-11-16T04:26:18,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/632c11c4abe24d89b72d38b813ae8a23, entries=150, sequenceid=433, filesize=12.0 K 2024-11-16T04:26:18,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/3f6d4b0beef94742999b32234e2238b9 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3f6d4b0beef94742999b32234e2238b9 2024-11-16T04:26:18,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3f6d4b0beef94742999b32234e2238b9, entries=150, sequenceid=433, filesize=12.0 K 2024-11-16T04:26:18,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3c5eac5e1b52fdf0c230183879dedd11 in 468ms, sequenceid=433, compaction requested=true 2024-11-16T04:26:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:18,719 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:18,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:18,720 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:18,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:18,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:18,723 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:18,723 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:18,723 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:18,723 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d7d81cf5fd874700a1e879cc0f0d8417, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6571177c784d40e2b99598826ae7b9fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/ec196e37be4145e5b634a84d373027be] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=39.3 K 2024-11-16T04:26:18,724 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:18,724 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:18,724 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:18,724 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/966f6a33441447aeb034161a17a5758c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b4afd2618424464da3df3130d62c60f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/632c11c4abe24d89b72d38b813ae8a23] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=36.9 K 2024-11-16T04:26:18,724 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7d81cf5fd874700a1e879cc0f0d8417, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731731176083 2024-11-16T04:26:18,724 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 966f6a33441447aeb034161a17a5758c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731731176083 2024-11-16T04:26:18,725 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6571177c784d40e2b99598826ae7b9fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731731176797 2024-11-16T04:26:18,727 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b4afd2618424464da3df3130d62c60f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731731176797 2024-11-16T04:26:18,727 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec196e37be4145e5b634a84d373027be, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731731178109 2024-11-16T04:26:18,729 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 632c11c4abe24d89b72d38b813ae8a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731731178119 2024-11-16T04:26:18,750 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:18,750 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:18,750 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/9433b7d3133343f3bdb28496e7e9cde2 is 50, key is test_row_0/B:col10/1731731178119/Put/seqid=0 2024-11-16T04:26:18,751 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/fa1203d4de9d4c7b886ef31511423358 is 50, key is test_row_0/A:col10/1731731178119/Put/seqid=0 2024-11-16T04:26:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:18,762 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:18,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:26:18,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742341_1517 (size=13323) 2024-11-16T04:26:18,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742342_1518 (size=13323) 2024-11-16T04:26:18,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/bcd5521c8f454394a53d18111f8e68b2 is 50, key is test_row_0/A:col10/1731731178302/Put/seqid=0 2024-11-16T04:26:18,771 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/9433b7d3133343f3bdb28496e7e9cde2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/9433b7d3133343f3bdb28496e7e9cde2 2024-11-16T04:26:18,773 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/fa1203d4de9d4c7b886ef31511423358 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/fa1203d4de9d4c7b886ef31511423358 2024-11-16T04:26:18,778 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 9433b7d3133343f3bdb28496e7e9cde2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:18,778 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,778 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731178719; duration=0sec 2024-11-16T04:26:18,778 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:18,778 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:18,779 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:18,780 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into fa1203d4de9d4c7b886ef31511423358(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:18,780 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,780 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731178719; duration=0sec 2024-11-16T04:26:18,780 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:18,780 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:18,780 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:18,780 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:18,781 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:18,781 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8035fdf363dc4c73aea80d8b1e9a9932, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b0fcf220c1e14216baafc21d528ef941, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3f6d4b0beef94742999b32234e2238b9] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=36.9 K 2024-11-16T04:26:18,784 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 8035fdf363dc4c73aea80d8b1e9a9932, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1731731176083 2024-11-16T04:26:18,789 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b0fcf220c1e14216baafc21d528ef941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731731176797 2024-11-16T04:26:18,789 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6d4b0beef94742999b32234e2238b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731731178119 2024-11-16T04:26:18,796 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#447 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:18,796 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/670501fb1c124be39ef2171c8f35b4ef is 50, key is test_row_0/C:col10/1731731178119/Put/seqid=0 2024-11-16T04:26:18,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742343_1519 (size=14741) 2024-11-16T04:26:18,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/bcd5521c8f454394a53d18111f8e68b2 2024-11-16T04:26:18,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742344_1520 (size=13323) 2024-11-16T04:26:18,811 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/670501fb1c124be39ef2171c8f35b4ef as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/670501fb1c124be39ef2171c8f35b4ef 2024-11-16T04:26:18,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8e7cc4c796894b70ad1337f0ba715a19 is 50, key is test_row_0/B:col10/1731731178302/Put/seqid=0 2024-11-16T04:26:18,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742345_1521 (size=12301) 2024-11-16T04:26:18,824 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 670501fb1c124be39ef2171c8f35b4ef(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:18,824 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:18,824 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731178720; duration=0sec 2024-11-16T04:26:18,824 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:18,824 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731238943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731238943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731239053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T04:26:19,123 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:19,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T04:26:19,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:19,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:19,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:19,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8e7cc4c796894b70ad1337f0ba715a19 2024-11-16T04:26:19,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/f310ad2e46d64b6597aac6a1fb485f06 is 50, key is test_row_0/C:col10/1731731178302/Put/seqid=0 2024-11-16T04:26:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742346_1522 (size=12301) 2024-11-16T04:26:19,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/f310ad2e46d64b6597aac6a1fb485f06 2024-11-16T04:26:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:19,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/bcd5521c8f454394a53d18111f8e68b2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bcd5521c8f454394a53d18111f8e68b2 2024-11-16T04:26:19,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bcd5521c8f454394a53d18111f8e68b2, entries=200, sequenceid=445, filesize=14.4 K 2024-11-16T04:26:19,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/8e7cc4c796894b70ad1337f0ba715a19 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8e7cc4c796894b70ad1337f0ba715a19 2024-11-16T04:26:19,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8e7cc4c796894b70ad1337f0ba715a19, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T04:26:19,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/f310ad2e46d64b6597aac6a1fb485f06 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f310ad2e46d64b6597aac6a1fb485f06 2024-11-16T04:26:19,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f310ad2e46d64b6597aac6a1fb485f06, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T04:26:19,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3c5eac5e1b52fdf0c230183879dedd11 in 501ms, sequenceid=445, compaction requested=false 2024-11-16T04:26:19,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:19,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:19,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:19,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:26:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/99efffd4a9014b0ab3a0a55c43f6be6c is 50, key is test_row_0/A:col10/1731731178941/Put/seqid=0 2024-11-16T04:26:19,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742347_1523 (size=14741) 2024-11-16T04:26:19,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731239328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:19,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:19,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:19,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:19,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731239435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:19,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:19,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731239463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:19,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731239653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:19,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/99efffd4a9014b0ab3a0a55c43f6be6c 2024-11-16T04:26:19,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/80f41309f08041289ef0ca863792ef55 is 50, key is test_row_0/B:col10/1731731178941/Put/seqid=0 2024-11-16T04:26:19,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742348_1524 (size=12301) 2024-11-16T04:26:19,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:19,792 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-16T04:26:19,792 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:19,792 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,792 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-16T04:26:19,792 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:19,792 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:19,792 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,792 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:19,794 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-16T04:26:19,794 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:19,794 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,795 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:19,801 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-16T04:26:19,801 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:19,801 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,801 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:19,802 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-16T04:26:19,802 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:19,802 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,802 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:19,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:19,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:19,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:19,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731239963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:19,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:20,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/80f41309f08041289ef0ca863792ef55 2024-11-16T04:26:20,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/dcaa00a60493457c981fd7a57e48a58d is 50, key is test_row_0/C:col10/1731731178941/Put/seqid=0 2024-11-16T04:26:20,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742349_1525 (size=12301) 2024-11-16T04:26:20,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:20,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:20,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58566 deadline: 1731731240474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:20,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58576 deadline: 1731731240474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:20,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:20,474 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in 3c5eac5e1b52fdf0c230183879dedd11 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 , the old value is region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:20,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., hostname=fda3efb342d8,45455,1731731034087, seqNum=2 because the exception is null or not the one we care about 2024-11-16T04:26:20,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/dcaa00a60493457c981fd7a57e48a58d 2024-11-16T04:26:20,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:20,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/99efffd4a9014b0ab3a0a55c43f6be6c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/99efffd4a9014b0ab3a0a55c43f6be6c 2024-11-16T04:26:20,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/99efffd4a9014b0ab3a0a55c43f6be6c, entries=200, sequenceid=473, filesize=14.4 K 2024-11-16T04:26:20,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/80f41309f08041289ef0ca863792ef55 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/80f41309f08041289ef0ca863792ef55 2024-11-16T04:26:20,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/80f41309f08041289ef0ca863792ef55, entries=150, sequenceid=473, filesize=12.0 K 2024-11-16T04:26:20,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/dcaa00a60493457c981fd7a57e48a58d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dcaa00a60493457c981fd7a57e48a58d 2024-11-16T04:26:20,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dcaa00a60493457c981fd7a57e48a58d, entries=150, sequenceid=473, filesize=12.0 K 2024-11-16T04:26:20,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3c5eac5e1b52fdf0c230183879dedd11 in 1238ms, sequenceid=473, compaction requested=true 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:20,516 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3c5eac5e1b52fdf0c230183879dedd11:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:20,516 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:20,517 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:20,517 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:20,517 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/A is initiating minor compaction (all files) 2024-11-16T04:26:20,517 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/B is initiating minor compaction (all files) 2024-11-16T04:26:20,517 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/A in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,517 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/B in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,517 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/fa1203d4de9d4c7b886ef31511423358, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bcd5521c8f454394a53d18111f8e68b2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/99efffd4a9014b0ab3a0a55c43f6be6c] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=41.8 K 2024-11-16T04:26:20,517 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/9433b7d3133343f3bdb28496e7e9cde2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8e7cc4c796894b70ad1337f0ba715a19, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/80f41309f08041289ef0ca863792ef55] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=37.0 K 2024-11-16T04:26:20,517 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9433b7d3133343f3bdb28496e7e9cde2, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731731178119 2024-11-16T04:26:20,517 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting fa1203d4de9d4c7b886ef31511423358, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731731178119 2024-11-16T04:26:20,518 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e7cc4c796894b70ad1337f0ba715a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731731178302 2024-11-16T04:26:20,518 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting bcd5521c8f454394a53d18111f8e68b2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731731178274 2024-11-16T04:26:20,518 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f41309f08041289ef0ca863792ef55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731731178906 2024-11-16T04:26:20,518 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 99efffd4a9014b0ab3a0a55c43f6be6c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731731178906 2024-11-16T04:26:20,522 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#B#compaction#453 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:20,523 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:20,523 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/07028acf6bff46ccb7f75527d0c7d1d3 is 50, key is test_row_0/B:col10/1731731178941/Put/seqid=0 2024-11-16T04:26:20,523 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/9b7c9d5082d9498093ab8789a4d4e4bc is 50, key is test_row_0/A:col10/1731731178941/Put/seqid=0 2024-11-16T04:26:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742351_1527 (size=13425) 2024-11-16T04:26:20,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742350_1526 (size=13425) 2024-11-16T04:26:20,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T04:26:20,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:26:20,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/11d15b209b1540d6870519c4579f0e1b is 50, key is test_row_0/A:col10/1731731179313/Put/seqid=0 2024-11-16T04:26:20,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742352_1528 (size=12301) 2024-11-16T04:26:20,933 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/07028acf6bff46ccb7f75527d0c7d1d3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/07028acf6bff46ccb7f75527d0c7d1d3 2024-11-16T04:26:20,933 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/9b7c9d5082d9498093ab8789a4d4e4bc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9b7c9d5082d9498093ab8789a4d4e4bc 2024-11-16T04:26:20,937 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/B of 3c5eac5e1b52fdf0c230183879dedd11 into 07028acf6bff46ccb7f75527d0c7d1d3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:20,937 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/A of 3c5eac5e1b52fdf0c230183879dedd11 into 9b7c9d5082d9498093ab8789a4d4e4bc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:20,937 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/B, priority=13, startTime=1731731180516; duration=0sec 2024-11-16T04:26:20,937 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/A, priority=13, startTime=1731731180516; duration=0sec 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:B 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:A 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:20,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): 3c5eac5e1b52fdf0c230183879dedd11/C is initiating minor compaction (all files) 2024-11-16T04:26:20,937 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3c5eac5e1b52fdf0c230183879dedd11/C in TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:20,938 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/670501fb1c124be39ef2171c8f35b4ef, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f310ad2e46d64b6597aac6a1fb485f06, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dcaa00a60493457c981fd7a57e48a58d] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp, totalSize=37.0 K 2024-11-16T04:26:20,938 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 670501fb1c124be39ef2171c8f35b4ef, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731731178119 2024-11-16T04:26:20,938 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f310ad2e46d64b6597aac6a1fb485f06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731731178302 2024-11-16T04:26:20,938 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcaa00a60493457c981fd7a57e48a58d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731731178906 2024-11-16T04:26:20,943 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3c5eac5e1b52fdf0c230183879dedd11#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:20,943 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/2fe5a23fc5314edfb815e3d7d2f0ff61 is 50, key is test_row_0/C:col10/1731731178941/Put/seqid=0 2024-11-16T04:26:20,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742353_1529 (size=13425) 2024-11-16T04:26:21,063 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/11d15b209b1540d6870519c4579f0e1b 2024-11-16T04:26:21,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/db37f623610c4a24ad6125170778073a is 50, key is test_row_0/B:col10/1731731179313/Put/seqid=0 2024-11-16T04:26:21,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742354_1530 (size=12301) 2024-11-16T04:26:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:21,350 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/2fe5a23fc5314edfb815e3d7d2f0ff61 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2fe5a23fc5314edfb815e3d7d2f0ff61 2024-11-16T04:26:21,354 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3c5eac5e1b52fdf0c230183879dedd11/C of 3c5eac5e1b52fdf0c230183879dedd11 into 2fe5a23fc5314edfb815e3d7d2f0ff61(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:21,354 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:21,354 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., storeName=3c5eac5e1b52fdf0c230183879dedd11/C, priority=13, startTime=1731731180516; duration=0sec 2024-11-16T04:26:21,354 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:21,354 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3c5eac5e1b52fdf0c230183879dedd11:C 2024-11-16T04:26:21,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/db37f623610c4a24ad6125170778073a 2024-11-16T04:26:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/1c2e9ccb9fb7431a9d09b431afbfa6bf is 50, key is test_row_0/C:col10/1731731179313/Put/seqid=0 2024-11-16T04:26:21,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742355_1531 (size=12301) 2024-11-16T04:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:21,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. as already flushing 2024-11-16T04:26:21,485 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-16T04:26:21,485 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:21,485 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:21,485 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:21,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:21,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/1c2e9ccb9fb7431a9d09b431afbfa6bf 2024-11-16T04:26:21,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/11d15b209b1540d6870519c4579f0e1b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/11d15b209b1540d6870519c4579f0e1b 2024-11-16T04:26:21,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/11d15b209b1540d6870519c4579f0e1b, entries=150, sequenceid=484, filesize=12.0 K 2024-11-16T04:26:21,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/db37f623610c4a24ad6125170778073a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/db37f623610c4a24ad6125170778073a 2024-11-16T04:26:21,892 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/db37f623610c4a24ad6125170778073a, entries=150, sequenceid=484, filesize=12.0 K 2024-11-16T04:26:21,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/1c2e9ccb9fb7431a9d09b431afbfa6bf as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/1c2e9ccb9fb7431a9d09b431afbfa6bf 2024-11-16T04:26:21,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/1c2e9ccb9fb7431a9d09b431afbfa6bf, entries=150, sequenceid=484, filesize=12.0 K 2024-11-16T04:26:21,896 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=6.71 KB/6870 for 3c5eac5e1b52fdf0c230183879dedd11 in 1239ms, sequenceid=484, compaction requested=false 2024-11-16T04:26:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 3c5eac5e1b52fdf0c230183879dedd11: 2024-11-16T04:26:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T04:26:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T04:26:21,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T04:26:21,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7730 sec 2024-11-16T04:26:21,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.7750 sec 2024-11-16T04:26:22,337 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T04:26:22,485 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-11-16T04:26:22,485 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:22,485 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:22,485 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:22,485 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T04:26:23,294 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:26,885 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-11-16T04:26:26,885 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:26,885 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:26,885 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:26,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:26,954 INFO [Thread-1921 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1921. 2024-11-16T04:26:26,954 DEBUG [Thread-1921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:26,954 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:26,954 DEBUG [Thread-1921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:26,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:28,305 INFO [Thread-1919 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1919. 2024-11-16T04:26:28,305 DEBUG [Thread-1919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:28,305 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:28,305 DEBUG [Thread-1919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:28,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:28,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T04:26:28,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-16T04:26:28,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-16T04:26:28,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-16T04:26:28,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-16T04:26:28,305 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1191 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3572 rows 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1188 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3562 rows 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1190 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3567 rows 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1179 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3531 rows 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1186 2024-11-16T04:26:28,306 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3555 rows 2024-11-16T04:26:28,306 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:26:28,306 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:26:28,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:28,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:28,306 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:28,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T04:26:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T04:26:28,310 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731188310"}]},"ts":"1731731188310"} 2024-11-16T04:26:28,311 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T04:26:28,311 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T04:26:28,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:26:28,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, UNASSIGN}] 2024-11-16T04:26:28,313 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, UNASSIGN 2024-11-16T04:26:28,314 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=3c5eac5e1b52fdf0c230183879dedd11, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:28,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, UNASSIGN because future has completed 2024-11-16T04:26:28,315 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:26:28,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:26:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T04:26:28,467 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:28,467 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:26:28,467 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 3c5eac5e1b52fdf0c230183879dedd11, disabling compactions & flushes 2024-11-16T04:26:28,467 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:28,467 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:28,467 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. after waiting 0 ms 2024-11-16T04:26:28,467 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:28,468 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing 3c5eac5e1b52fdf0c230183879dedd11 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-16T04:26:28,472 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/e836deb086394673a2efa9d0bbc4d4ea is 50, key is test_row_1/A:col10/1731731188304/Put/seqid=0 2024-11-16T04:26:28,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742356_1532 (size=9857) 2024-11-16T04:26:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T04:26:28,878 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/e836deb086394673a2efa9d0bbc4d4ea 2024-11-16T04:26:28,893 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/e96a992adbef49f583921412ed7a7aa3 is 50, key is test_row_1/B:col10/1731731188304/Put/seqid=0 2024-11-16T04:26:28,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742357_1533 (size=9857) 2024-11-16T04:26:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T04:26:29,298 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/e96a992adbef49f583921412ed7a7aa3 2024-11-16T04:26:29,312 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/f47736df96684355affb616bd39aa5d0 is 50, key is test_row_1/C:col10/1731731188304/Put/seqid=0 2024-11-16T04:26:29,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742358_1534 (size=9857) 2024-11-16T04:26:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T04:26:29,719 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/f47736df96684355affb616bd39aa5d0 2024-11-16T04:26:29,728 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/A/e836deb086394673a2efa9d0bbc4d4ea as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e836deb086394673a2efa9d0bbc4d4ea 2024-11-16T04:26:29,733 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e836deb086394673a2efa9d0bbc4d4ea, entries=100, sequenceid=495, filesize=9.6 K 2024-11-16T04:26:29,734 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/B/e96a992adbef49f583921412ed7a7aa3 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/e96a992adbef49f583921412ed7a7aa3 2024-11-16T04:26:29,738 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/e96a992adbef49f583921412ed7a7aa3, entries=100, sequenceid=495, filesize=9.6 K 2024-11-16T04:26:29,739 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/.tmp/C/f47736df96684355affb616bd39aa5d0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f47736df96684355affb616bd39aa5d0 2024-11-16T04:26:29,742 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f47736df96684355affb616bd39aa5d0, entries=100, sequenceid=495, filesize=9.6 K 2024-11-16T04:26:29,743 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3c5eac5e1b52fdf0c230183879dedd11 in 1276ms, sequenceid=495, compaction requested=true 2024-11-16T04:26:29,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3aabb17fa608407d983e22782cf80095, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/13ec4a7348a64925910952992c206abc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/a28a7b9b54c44ac28394cc660e3cca24, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/b2760d7ff0934819b486894252980d2d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/603eafdb7695460891ab0704ccbed61c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8190fed7a7c2414598bb558c152db1b1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/90349fac7460495eab68dce41ad51ce2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/0791d03dfe5f4c338a961f683c95eac6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9c41e659a37e47a6b789f6890b19e047, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6f86a75bf9d947638bdb9bcb922233b7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/7aecb901fd604d2e9f97dd7bcda00bcf, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/dbb9cff47b3e4686b698c3433947f7f8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d91248b9e0524348946333844bdaaf1e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c219aa7bcc1246f0805acc0c27bb6f49, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/2c271e7ab11d44d9b6c1e6425dbf4731, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/68f87c553a83407abaea721756487d2d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8343381f9e1444e5baa793a1f6f152db, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bf9cf9b90c7746a9960c78144bcad24d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/5e688391903045e490674a49b4108e71, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/10fdc0d3a0f24686a9b1441345982d81, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d8387a0e3ffa40598c5e9a4f4804606d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/85852223c36c490bb0165a6808cb7b96, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e7f70d023e5346539182e34ef636b5e6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/643baceb0ccf4c88b202138db8df9b9c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6817df2ec0fd4b599db03c8ea954f247, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c4b6bd7de405482eb0d7411181f64524, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d7d81cf5fd874700a1e879cc0f0d8417, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6571177c784d40e2b99598826ae7b9fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/ec196e37be4145e5b634a84d373027be, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/fa1203d4de9d4c7b886ef31511423358, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bcd5521c8f454394a53d18111f8e68b2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/99efffd4a9014b0ab3a0a55c43f6be6c] to archive 2024-11-16T04:26:29,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:26:29,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3aabb17fa608407d983e22782cf80095 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3aabb17fa608407d983e22782cf80095 2024-11-16T04:26:29,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/13ec4a7348a64925910952992c206abc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/13ec4a7348a64925910952992c206abc 2024-11-16T04:26:29,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/a28a7b9b54c44ac28394cc660e3cca24 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/a28a7b9b54c44ac28394cc660e3cca24 2024-11-16T04:26:29,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/b2760d7ff0934819b486894252980d2d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/b2760d7ff0934819b486894252980d2d 2024-11-16T04:26:29,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/603eafdb7695460891ab0704ccbed61c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/603eafdb7695460891ab0704ccbed61c 2024-11-16T04:26:29,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8190fed7a7c2414598bb558c152db1b1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8190fed7a7c2414598bb558c152db1b1 2024-11-16T04:26:29,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/90349fac7460495eab68dce41ad51ce2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/90349fac7460495eab68dce41ad51ce2 2024-11-16T04:26:29,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/0791d03dfe5f4c338a961f683c95eac6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/0791d03dfe5f4c338a961f683c95eac6 2024-11-16T04:26:29,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9c41e659a37e47a6b789f6890b19e047 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9c41e659a37e47a6b789f6890b19e047 2024-11-16T04:26:29,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6f86a75bf9d947638bdb9bcb922233b7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6f86a75bf9d947638bdb9bcb922233b7 2024-11-16T04:26:29,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/7aecb901fd604d2e9f97dd7bcda00bcf to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/7aecb901fd604d2e9f97dd7bcda00bcf 2024-11-16T04:26:29,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/dbb9cff47b3e4686b698c3433947f7f8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/dbb9cff47b3e4686b698c3433947f7f8 2024-11-16T04:26:29,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d91248b9e0524348946333844bdaaf1e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d91248b9e0524348946333844bdaaf1e 2024-11-16T04:26:29,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c219aa7bcc1246f0805acc0c27bb6f49 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c219aa7bcc1246f0805acc0c27bb6f49 2024-11-16T04:26:29,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/2c271e7ab11d44d9b6c1e6425dbf4731 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/2c271e7ab11d44d9b6c1e6425dbf4731 2024-11-16T04:26:29,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/68f87c553a83407abaea721756487d2d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/68f87c553a83407abaea721756487d2d 2024-11-16T04:26:29,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8343381f9e1444e5baa793a1f6f152db to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/8343381f9e1444e5baa793a1f6f152db 2024-11-16T04:26:29,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bf9cf9b90c7746a9960c78144bcad24d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bf9cf9b90c7746a9960c78144bcad24d 2024-11-16T04:26:29,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/5e688391903045e490674a49b4108e71 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/5e688391903045e490674a49b4108e71 2024-11-16T04:26:29,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/10fdc0d3a0f24686a9b1441345982d81 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/10fdc0d3a0f24686a9b1441345982d81 2024-11-16T04:26:29,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/3a99c9fa0f2e4a95b50ff7ade3ef2fbc 2024-11-16T04:26:29,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d8387a0e3ffa40598c5e9a4f4804606d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d8387a0e3ffa40598c5e9a4f4804606d 2024-11-16T04:26:29,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/85852223c36c490bb0165a6808cb7b96 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/85852223c36c490bb0165a6808cb7b96 2024-11-16T04:26:29,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e7f70d023e5346539182e34ef636b5e6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e7f70d023e5346539182e34ef636b5e6 2024-11-16T04:26:29,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/643baceb0ccf4c88b202138db8df9b9c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/643baceb0ccf4c88b202138db8df9b9c 2024-11-16T04:26:29,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6817df2ec0fd4b599db03c8ea954f247 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6817df2ec0fd4b599db03c8ea954f247 2024-11-16T04:26:29,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c4b6bd7de405482eb0d7411181f64524 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/c4b6bd7de405482eb0d7411181f64524 2024-11-16T04:26:29,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d7d81cf5fd874700a1e879cc0f0d8417 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/d7d81cf5fd874700a1e879cc0f0d8417 2024-11-16T04:26:29,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6571177c784d40e2b99598826ae7b9fb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/6571177c784d40e2b99598826ae7b9fb 2024-11-16T04:26:29,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/ec196e37be4145e5b634a84d373027be to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/ec196e37be4145e5b634a84d373027be 2024-11-16T04:26:29,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/fa1203d4de9d4c7b886ef31511423358 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/fa1203d4de9d4c7b886ef31511423358 2024-11-16T04:26:29,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bcd5521c8f454394a53d18111f8e68b2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/bcd5521c8f454394a53d18111f8e68b2 2024-11-16T04:26:29,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/99efffd4a9014b0ab3a0a55c43f6be6c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/99efffd4a9014b0ab3a0a55c43f6be6c 2024-11-16T04:26:29,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/22b422b1392541c484490c29c5b8b6e0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/d8e4fd87d7b244faab24de50315b84b9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b451306883614f0aa476dff908afc658, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/5b3c564ce5d0488ca2b41cd0af05d005, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/4593f6e719094f57953f9efc3b186de2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c5137fc286574863b2d2404ff1c5365c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8df3242421a1415c8217128ab5cac953, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a0de66effe8f4c658f681e0d690799cf, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/934297517b9d4bd481ef8652cb9d2636, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8757205544464cc5a582ef9744674701, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/95117a10737147b9a9a8d8e806747235, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1900d42858fe47718962bff7d8e7d6b8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/0c5ba5193ed44da79d8acdac85deeb05, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/72ab90a4174c4e95a0be62474136a828, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/f9ce7ce6ee16464c8e8305c65db5b0a6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c6914644b7be444391d85bffda9c6f63, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/182f7372b65445f3b2d568975f5b56c1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/dd44bdef461f4958b1bbaa52dd4e8d86, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/708fc586bdf2470da2e2185168dad108, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/be5410f233424cad9928cec895389ec5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/749a4880800941e3b7ce04038cdc2cfd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/20b92d2cb8e642d299e373658a5c970d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/2ffc149d33fd41f18d53f93089d778f1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/14c8de261de94dd589951be5834f54e8, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8c0c07a3bc4847afb4d60e8b3759f9e7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a5fc4ef05e3e4d7b822f81feadb49349, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/966f6a33441447aeb034161a17a5758c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1f1025b42ea249e19ebb75bc19d298ea, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b4afd2618424464da3df3130d62c60f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/9433b7d3133343f3bdb28496e7e9cde2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/632c11c4abe24d89b72d38b813ae8a23, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8e7cc4c796894b70ad1337f0ba715a19, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/80f41309f08041289ef0ca863792ef55] to archive 2024-11-16T04:26:29,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:26:29,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/22b422b1392541c484490c29c5b8b6e0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/22b422b1392541c484490c29c5b8b6e0 2024-11-16T04:26:29,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/d8e4fd87d7b244faab24de50315b84b9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/d8e4fd87d7b244faab24de50315b84b9 2024-11-16T04:26:29,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b451306883614f0aa476dff908afc658 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b451306883614f0aa476dff908afc658 2024-11-16T04:26:29,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/5b3c564ce5d0488ca2b41cd0af05d005 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/5b3c564ce5d0488ca2b41cd0af05d005 2024-11-16T04:26:29,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/4593f6e719094f57953f9efc3b186de2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/4593f6e719094f57953f9efc3b186de2 2024-11-16T04:26:29,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c5137fc286574863b2d2404ff1c5365c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c5137fc286574863b2d2404ff1c5365c 2024-11-16T04:26:29,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8df3242421a1415c8217128ab5cac953 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8df3242421a1415c8217128ab5cac953 2024-11-16T04:26:29,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a0de66effe8f4c658f681e0d690799cf to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a0de66effe8f4c658f681e0d690799cf 2024-11-16T04:26:29,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/934297517b9d4bd481ef8652cb9d2636 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/934297517b9d4bd481ef8652cb9d2636 2024-11-16T04:26:29,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8757205544464cc5a582ef9744674701 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8757205544464cc5a582ef9744674701 2024-11-16T04:26:29,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/95117a10737147b9a9a8d8e806747235 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/95117a10737147b9a9a8d8e806747235 2024-11-16T04:26:29,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1900d42858fe47718962bff7d8e7d6b8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1900d42858fe47718962bff7d8e7d6b8 2024-11-16T04:26:29,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/0c5ba5193ed44da79d8acdac85deeb05 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/0c5ba5193ed44da79d8acdac85deeb05 2024-11-16T04:26:29,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/72ab90a4174c4e95a0be62474136a828 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/72ab90a4174c4e95a0be62474136a828 2024-11-16T04:26:29,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/f9ce7ce6ee16464c8e8305c65db5b0a6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/f9ce7ce6ee16464c8e8305c65db5b0a6 2024-11-16T04:26:29,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c6914644b7be444391d85bffda9c6f63 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/c6914644b7be444391d85bffda9c6f63 2024-11-16T04:26:29,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/182f7372b65445f3b2d568975f5b56c1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/182f7372b65445f3b2d568975f5b56c1 2024-11-16T04:26:29,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/dd44bdef461f4958b1bbaa52dd4e8d86 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/dd44bdef461f4958b1bbaa52dd4e8d86 2024-11-16T04:26:29,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/708fc586bdf2470da2e2185168dad108 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/708fc586bdf2470da2e2185168dad108 2024-11-16T04:26:29,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/be5410f233424cad9928cec895389ec5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/be5410f233424cad9928cec895389ec5 2024-11-16T04:26:29,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/749a4880800941e3b7ce04038cdc2cfd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/749a4880800941e3b7ce04038cdc2cfd 2024-11-16T04:26:29,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/20b92d2cb8e642d299e373658a5c970d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/20b92d2cb8e642d299e373658a5c970d 2024-11-16T04:26:29,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/2ffc149d33fd41f18d53f93089d778f1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/2ffc149d33fd41f18d53f93089d778f1 2024-11-16T04:26:29,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/14c8de261de94dd589951be5834f54e8 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/14c8de261de94dd589951be5834f54e8 2024-11-16T04:26:29,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8c0c07a3bc4847afb4d60e8b3759f9e7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8c0c07a3bc4847afb4d60e8b3759f9e7 2024-11-16T04:26:29,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a5fc4ef05e3e4d7b822f81feadb49349 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/a5fc4ef05e3e4d7b822f81feadb49349 2024-11-16T04:26:29,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/966f6a33441447aeb034161a17a5758c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/966f6a33441447aeb034161a17a5758c 2024-11-16T04:26:29,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1f1025b42ea249e19ebb75bc19d298ea to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/1f1025b42ea249e19ebb75bc19d298ea 2024-11-16T04:26:29,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b4afd2618424464da3df3130d62c60f2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/b4afd2618424464da3df3130d62c60f2 2024-11-16T04:26:29,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/9433b7d3133343f3bdb28496e7e9cde2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/9433b7d3133343f3bdb28496e7e9cde2 2024-11-16T04:26:29,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/632c11c4abe24d89b72d38b813ae8a23 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/632c11c4abe24d89b72d38b813ae8a23 2024-11-16T04:26:29,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8e7cc4c796894b70ad1337f0ba715a19 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/8e7cc4c796894b70ad1337f0ba715a19 2024-11-16T04:26:29,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/80f41309f08041289ef0ca863792ef55 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/80f41309f08041289ef0ca863792ef55 2024-11-16T04:26:29,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2b5869e3b8aa4cd0bee2ab2699da6296, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/292c14072da4436d8778209464b7ad8f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3905320817fa4447977d88b6abd46c47, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d706972f18bd4afabe8e766aaac6d6c4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/000a4f9db40949ac9828893d1e476383, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/668820a842d9408aa0aaea4e2203d759, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/9990dcfa024a4b8fa6270c5887c94fdd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/a4dec5a7107d4636848d4e2950384a6f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d7e7f511316948a0991c8de030e6fc4f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/6be8c97fc4f34c1993f08b5fba06dab1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/272c1407e3bb49d7bd26e76ab5da58cd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ce7419cbacf34b51ad335829d39cde1b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b3b71424a60140aa86aac058a21a884c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/22d71c72f87243538b70df7766ca7923, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/394918ecab3c450eb7c8a89033e6ac56, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/01813ac495ec49bf8d4ae634cb6ecd0f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/0b5140be63ea4c98a4b844cfd6ebd9ce, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/fbb62ec4633d4845aab98c7f5f74a68c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/c45d32f8deed4a279493f8b8feadbf7c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/997536ac45eb48c59a99e2e9bc653c01, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/82a61b2e3e40426f95d413ee275512f2, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/419b5f408df84090be49acb34a27db3e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d032be82417a4b46910eda3c29e466af, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dbac8e99c30d491ca069bce8267c91fb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/60ec2f5a638c4218babc5e1db4ba8afe, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8b79a4891f074e1aab3e939524d11a6a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8035fdf363dc4c73aea80d8b1e9a9932, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ef3ebe40cbe246d88a46d77973740469, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b0fcf220c1e14216baafc21d528ef941, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/670501fb1c124be39ef2171c8f35b4ef, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3f6d4b0beef94742999b32234e2238b9, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f310ad2e46d64b6597aac6a1fb485f06, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dcaa00a60493457c981fd7a57e48a58d] to archive 2024-11-16T04:26:29,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:26:29,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2b5869e3b8aa4cd0bee2ab2699da6296 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2b5869e3b8aa4cd0bee2ab2699da6296 2024-11-16T04:26:29,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/292c14072da4436d8778209464b7ad8f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/292c14072da4436d8778209464b7ad8f 2024-11-16T04:26:29,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3905320817fa4447977d88b6abd46c47 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3905320817fa4447977d88b6abd46c47 2024-11-16T04:26:29,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d706972f18bd4afabe8e766aaac6d6c4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d706972f18bd4afabe8e766aaac6d6c4 2024-11-16T04:26:29,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/000a4f9db40949ac9828893d1e476383 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/000a4f9db40949ac9828893d1e476383 2024-11-16T04:26:29,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/668820a842d9408aa0aaea4e2203d759 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/668820a842d9408aa0aaea4e2203d759 2024-11-16T04:26:29,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/9990dcfa024a4b8fa6270c5887c94fdd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/9990dcfa024a4b8fa6270c5887c94fdd 2024-11-16T04:26:29,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/a4dec5a7107d4636848d4e2950384a6f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/a4dec5a7107d4636848d4e2950384a6f 2024-11-16T04:26:29,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d7e7f511316948a0991c8de030e6fc4f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d7e7f511316948a0991c8de030e6fc4f 2024-11-16T04:26:29,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/6be8c97fc4f34c1993f08b5fba06dab1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/6be8c97fc4f34c1993f08b5fba06dab1 2024-11-16T04:26:29,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/272c1407e3bb49d7bd26e76ab5da58cd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/272c1407e3bb49d7bd26e76ab5da58cd 2024-11-16T04:26:29,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ce7419cbacf34b51ad335829d39cde1b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ce7419cbacf34b51ad335829d39cde1b 2024-11-16T04:26:29,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b3b71424a60140aa86aac058a21a884c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b3b71424a60140aa86aac058a21a884c 2024-11-16T04:26:29,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/22d71c72f87243538b70df7766ca7923 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/22d71c72f87243538b70df7766ca7923 2024-11-16T04:26:29,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/394918ecab3c450eb7c8a89033e6ac56 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/394918ecab3c450eb7c8a89033e6ac56 2024-11-16T04:26:29,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/01813ac495ec49bf8d4ae634cb6ecd0f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/01813ac495ec49bf8d4ae634cb6ecd0f 2024-11-16T04:26:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/0b5140be63ea4c98a4b844cfd6ebd9ce to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/0b5140be63ea4c98a4b844cfd6ebd9ce 2024-11-16T04:26:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/fbb62ec4633d4845aab98c7f5f74a68c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/fbb62ec4633d4845aab98c7f5f74a68c 2024-11-16T04:26:29,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/c45d32f8deed4a279493f8b8feadbf7c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/c45d32f8deed4a279493f8b8feadbf7c 2024-11-16T04:26:29,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/997536ac45eb48c59a99e2e9bc653c01 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/997536ac45eb48c59a99e2e9bc653c01 2024-11-16T04:26:29,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/82a61b2e3e40426f95d413ee275512f2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/82a61b2e3e40426f95d413ee275512f2 2024-11-16T04:26:29,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/419b5f408df84090be49acb34a27db3e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/419b5f408df84090be49acb34a27db3e 2024-11-16T04:26:29,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d032be82417a4b46910eda3c29e466af to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/d032be82417a4b46910eda3c29e466af 2024-11-16T04:26:29,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dbac8e99c30d491ca069bce8267c91fb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dbac8e99c30d491ca069bce8267c91fb 2024-11-16T04:26:29,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/60ec2f5a638c4218babc5e1db4ba8afe to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/60ec2f5a638c4218babc5e1db4ba8afe 2024-11-16T04:26:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8b79a4891f074e1aab3e939524d11a6a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8b79a4891f074e1aab3e939524d11a6a 2024-11-16T04:26:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8035fdf363dc4c73aea80d8b1e9a9932 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/8035fdf363dc4c73aea80d8b1e9a9932 2024-11-16T04:26:29,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ef3ebe40cbe246d88a46d77973740469 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/ef3ebe40cbe246d88a46d77973740469 2024-11-16T04:26:29,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b0fcf220c1e14216baafc21d528ef941 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/b0fcf220c1e14216baafc21d528ef941 2024-11-16T04:26:29,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/670501fb1c124be39ef2171c8f35b4ef to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/670501fb1c124be39ef2171c8f35b4ef 2024-11-16T04:26:29,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3f6d4b0beef94742999b32234e2238b9 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/3f6d4b0beef94742999b32234e2238b9 2024-11-16T04:26:29,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f310ad2e46d64b6597aac6a1fb485f06 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f310ad2e46d64b6597aac6a1fb485f06 2024-11-16T04:26:29,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dcaa00a60493457c981fd7a57e48a58d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/dcaa00a60493457c981fd7a57e48a58d 2024-11-16T04:26:29,828 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-16T04:26:29,829 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. 2024-11-16T04:26:29,829 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 3c5eac5e1b52fdf0c230183879dedd11: Waiting for close lock at 1731731188467Running coprocessor pre-close hooks at 1731731188467Disabling compacts and flushes for region at 1731731188467Disabling writes for close at 1731731188467Obtaining lock to block concurrent updates at 1731731188468 (+1 ms)Preparing flush snapshotting stores in 3c5eac5e1b52fdf0c230183879dedd11 at 1731731188468Finished memstore snapshotting TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731731188468Flushing stores of TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11. at 1731731188468Flushing 3c5eac5e1b52fdf0c230183879dedd11/A: creating writer at 1731731188468Flushing 3c5eac5e1b52fdf0c230183879dedd11/A: appending metadata at 1731731188471 (+3 ms)Flushing 3c5eac5e1b52fdf0c230183879dedd11/A: closing flushed file at 1731731188471Flushing 3c5eac5e1b52fdf0c230183879dedd11/B: creating writer at 1731731188887 (+416 ms)Flushing 3c5eac5e1b52fdf0c230183879dedd11/B: appending metadata at 1731731188892 (+5 ms)Flushing 3c5eac5e1b52fdf0c230183879dedd11/B: closing flushed file at 1731731188892Flushing 3c5eac5e1b52fdf0c230183879dedd11/C: creating writer at 1731731189307 (+415 ms)Flushing 3c5eac5e1b52fdf0c230183879dedd11/C: appending metadata at 1731731189311 (+4 ms)Flushing 3c5eac5e1b52fdf0c230183879dedd11/C: closing flushed file at 1731731189311Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71138e90: reopening flushed file at 1731731189727 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ecd177: reopening flushed file at 1731731189733 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66807ba8: reopening flushed file at 1731731189738 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 3c5eac5e1b52fdf0c230183879dedd11 in 1276ms, sequenceid=495, compaction requested=true at 1731731189743 (+5 ms)Writing region close event to WAL at 1731731189826 (+83 ms)Running coprocessor post-close hooks at 1731731189828 (+2 ms)Closed at 1731731189829 (+1 ms) 2024-11-16T04:26:29,830 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:29,830 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=3c5eac5e1b52fdf0c230183879dedd11, regionState=CLOSED 2024-11-16T04:26:29,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:26:29,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T04:26:29,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 3c5eac5e1b52fdf0c230183879dedd11, server=fda3efb342d8,45455,1731731034087 in 1.5170 sec 2024-11-16T04:26:29,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-16T04:26:29,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3c5eac5e1b52fdf0c230183879dedd11, UNASSIGN in 1.5200 sec 2024-11-16T04:26:29,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T04:26:29,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5220 sec 2024-11-16T04:26:29,837 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731189836"}]},"ts":"1731731189836"} 2024-11-16T04:26:29,838 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T04:26:29,838 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T04:26:29,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5310 sec 2024-11-16T04:26:30,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T04:26:30,455 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:30,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T04:26:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,462 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T04:26:30,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:30,469 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/recovered.edits] 2024-11-16T04:26:30,473 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/11d15b209b1540d6870519c4579f0e1b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/11d15b209b1540d6870519c4579f0e1b 2024-11-16T04:26:30,474 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9b7c9d5082d9498093ab8789a4d4e4bc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/9b7c9d5082d9498093ab8789a4d4e4bc 2024-11-16T04:26:30,475 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e836deb086394673a2efa9d0bbc4d4ea to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/A/e836deb086394673a2efa9d0bbc4d4ea 2024-11-16T04:26:30,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/07028acf6bff46ccb7f75527d0c7d1d3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/07028acf6bff46ccb7f75527d0c7d1d3 2024-11-16T04:26:30,479 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/db37f623610c4a24ad6125170778073a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/db37f623610c4a24ad6125170778073a 2024-11-16T04:26:30,481 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/e96a992adbef49f583921412ed7a7aa3 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/B/e96a992adbef49f583921412ed7a7aa3 2024-11-16T04:26:30,483 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/1c2e9ccb9fb7431a9d09b431afbfa6bf to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/1c2e9ccb9fb7431a9d09b431afbfa6bf 2024-11-16T04:26:30,485 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2fe5a23fc5314edfb815e3d7d2f0ff61 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/2fe5a23fc5314edfb815e3d7d2f0ff61 2024-11-16T04:26:30,486 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f47736df96684355affb616bd39aa5d0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/C/f47736df96684355affb616bd39aa5d0 2024-11-16T04:26:30,489 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/recovered.edits/498.seqid to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11/recovered.edits/498.seqid 2024-11-16T04:26:30,489 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/3c5eac5e1b52fdf0c230183879dedd11 2024-11-16T04:26:30,489 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T04:26:30,492 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,494 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T04:26:30,497 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T04:26:30,499 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,499 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T04:26:30,499 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731731190499"}]},"ts":"9223372036854775807"} 2024-11-16T04:26:30,502 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T04:26:30,502 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3c5eac5e1b52fdf0c230183879dedd11, NAME => 'TestAcidGuarantees,,1731731157594.3c5eac5e1b52fdf0c230183879dedd11.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T04:26:30,502 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T04:26:30,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731731190502"}]},"ts":"9223372036854775807"} 2024-11-16T04:26:30,504 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T04:26:30,505 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-16T04:26:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T04:26:30,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T04:26:30,574 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:30,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 240), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=432 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=7995 (was 8068) 2024-11-16T04:26:30,598 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=432, ProcessCount=11, AvailableMemoryMB=7995 2024-11-16T04:26:30,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:26:30,599 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:26:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:30,601 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T04:26:30,601 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:30,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-16T04:26:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T04:26:30,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T04:26:30,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742359_1535 (size=959) 2024-11-16T04:26:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T04:26:30,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T04:26:31,012 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd 2024-11-16T04:26:31,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742360_1536 (size=53) 2024-11-16T04:26:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T04:26:31,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:26:31,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c16a25941e3d02bc3deccfcca062c9f6, disabling compactions & flushes 2024-11-16T04:26:31,421 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. after waiting 0 ms 2024-11-16T04:26:31,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,421 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,421 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c16a25941e3d02bc3deccfcca062c9f6: Waiting for close lock at 1731731191421Disabling compacts and flushes for region at 1731731191421Disabling writes for close at 1731731191421Writing region close event to WAL at 1731731191421Closed at 1731731191421 2024-11-16T04:26:31,422 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T04:26:31,423 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731731191422"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731731191422"}]},"ts":"1731731191422"} 2024-11-16T04:26:31,425 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T04:26:31,426 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T04:26:31,426 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731191426"}]},"ts":"1731731191426"} 2024-11-16T04:26:31,427 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T04:26:31,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, ASSIGN}] 2024-11-16T04:26:31,429 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, ASSIGN 2024-11-16T04:26:31,431 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, ASSIGN; state=OFFLINE, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=false 2024-11-16T04:26:31,581 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:31,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, ASSIGN because future has completed 2024-11-16T04:26:31,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:26:31,742 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,743 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:26:31,744 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,744 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:26:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T04:26:31,744 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,744 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,747 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,749 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c16a25941e3d02bc3deccfcca062c9f6 columnFamilyName A 2024-11-16T04:26:31,749 DEBUG [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:31,749 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(327): Store=c16a25941e3d02bc3deccfcca062c9f6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:26:31,750 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,751 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c16a25941e3d02bc3deccfcca062c9f6 columnFamilyName B 2024-11-16T04:26:31,751 DEBUG [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:31,752 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(327): Store=c16a25941e3d02bc3deccfcca062c9f6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:26:31,752 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,753 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c16a25941e3d02bc3deccfcca062c9f6 columnFamilyName C 2024-11-16T04:26:31,754 DEBUG [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:31,754 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(327): Store=c16a25941e3d02bc3deccfcca062c9f6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:26:31,754 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,755 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,756 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,758 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,758 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,758 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:26:31,760 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,763 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T04:26:31,764 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened c16a25941e3d02bc3deccfcca062c9f6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58905142, jitterRate=-0.12224498391151428}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:26:31,764 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:31,765 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for c16a25941e3d02bc3deccfcca062c9f6: Running coprocessor pre-open hook at 1731731191744Writing region info on filesystem at 1731731191744Initializing all the Stores at 1731731191746 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731191746Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731191747 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731191747Cleaning up temporary data from old regions at 1731731191758 (+11 ms)Running coprocessor post-open hooks at 1731731191764 (+6 ms)Region opened successfully at 1731731191764 2024-11-16T04:26:31,766 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., pid=142, masterSystemTime=1731731191736 2024-11-16T04:26:31,768 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,768 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:31,769 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=OPEN, openSeqNum=2, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:31,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:26:31,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T04:26:31,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 in 189 msec 2024-11-16T04:26:31,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-16T04:26:31,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, ASSIGN in 348 msec 2024-11-16T04:26:31,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T04:26:31,780 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731191780"}]},"ts":"1731731191780"} 2024-11-16T04:26:31,782 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T04:26:31,783 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T04:26:31,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1840 sec 2024-11-16T04:26:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-16T04:26:32,765 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:32,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173ad90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:32,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:32,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:32,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef4c539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:32,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:32,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:32,775 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c43f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:32,777 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:32,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:32,780 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T04:26:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fda3efb342d8,45087,1731731033310 2024-11-16T04:26:32,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3ce2f508 2024-11-16T04:26:32,782 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T04:26:32,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T04:26:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T04:26:32,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T04:26:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:32,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742361_1537 (size=995) 2024-11-16T04:26:33,201 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-16T04:26:33,201 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-16T04:26:33,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:26:33,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, REOPEN/MOVE}] 2024-11-16T04:26:33,209 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, REOPEN/MOVE 2024-11-16T04:26:33,209 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, REOPEN/MOVE because future has completed 2024-11-16T04:26:33,211 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:26:33,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:26:33,364 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,365 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:26:33,365 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing c16a25941e3d02bc3deccfcca062c9f6, disabling compactions & flushes 2024-11-16T04:26:33,365 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,365 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,365 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. after waiting 0 ms 2024-11-16T04:26:33,365 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,407 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T04:26:33,409 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,409 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for c16a25941e3d02bc3deccfcca062c9f6: Waiting for close lock at 1731731193365Running coprocessor pre-close hooks at 1731731193365Disabling compacts and flushes for region at 1731731193365Disabling writes for close at 1731731193365Writing region close event to WAL at 1731731193366 (+1 ms)Running coprocessor post-close hooks at 1731731193409 (+43 ms)Closed at 1731731193409 2024-11-16T04:26:33,409 WARN [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: c16a25941e3d02bc3deccfcca062c9f6 to self. 2024-11-16T04:26:33,414 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,416 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=CLOSED 2024-11-16T04:26:33,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:26:33,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-16T04:26:33,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 in 208 msec 2024-11-16T04:26:33,423 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, REOPEN/MOVE; state=CLOSED, location=fda3efb342d8,45455,1731731034087; forceNewPlan=false, retain=true 2024-11-16T04:26:33,574 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=OPENING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, REOPEN/MOVE because future has completed 2024-11-16T04:26:33,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:26:33,668 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T04:26:33,738 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,739 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} 2024-11-16T04:26:33,739 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,739 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T04:26:33,740 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,740 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,743 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,744 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c16a25941e3d02bc3deccfcca062c9f6 columnFamilyName A 2024-11-16T04:26:33,745 DEBUG [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:33,745 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(327): Store=c16a25941e3d02bc3deccfcca062c9f6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:26:33,746 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,746 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c16a25941e3d02bc3deccfcca062c9f6 columnFamilyName B 2024-11-16T04:26:33,746 DEBUG [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:33,747 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(327): Store=c16a25941e3d02bc3deccfcca062c9f6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:26:33,747 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,747 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c16a25941e3d02bc3deccfcca062c9f6 columnFamilyName C 2024-11-16T04:26:33,747 DEBUG [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:33,748 INFO [StoreOpener-c16a25941e3d02bc3deccfcca062c9f6-1 {}] regionserver.HStore(327): Store=c16a25941e3d02bc3deccfcca062c9f6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T04:26:33,748 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,748 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,749 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,750 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,750 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,750 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T04:26:33,751 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,752 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened c16a25941e3d02bc3deccfcca062c9f6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72280037, jitterRate=0.07705648243427277}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T04:26:33,752 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,752 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for c16a25941e3d02bc3deccfcca062c9f6: Running coprocessor pre-open hook at 1731731193740Writing region info on filesystem at 1731731193740Initializing all the Stores at 1731731193742 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731193742Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731193742Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731731193742Cleaning up temporary data from old regions at 1731731193750 (+8 ms)Running coprocessor post-open hooks at 1731731193752 (+2 ms)Region opened successfully at 1731731193752 2024-11-16T04:26:33,753 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., pid=147, masterSystemTime=1731731193732 2024-11-16T04:26:33,754 DEBUG [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,754 INFO [RS_OPEN_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,755 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=OPEN, openSeqNum=5, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:26:33,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-16T04:26:33,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 in 178 msec 2024-11-16T04:26:33,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-16T04:26:33,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, REOPEN/MOVE in 550 msec 2024-11-16T04:26:33,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T04:26:33,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 554 msec 2024-11-16T04:26:33,762 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 976 msec 2024-11-16T04:26:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-16T04:26:33,763 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:33,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52dabf58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,764 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@747beb37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,765 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52ea1e68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4262b017, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,767 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cefda21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,768 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e955da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d6b3ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,769 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dcfd212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,770 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4b482b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263b0516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db37d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,772 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc248c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792e8aea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,774 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796ffd3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,775 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b88089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@580223ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,777 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6406fe52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@81bb540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e638e47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aac1514, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,781 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494cb24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42aa7a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156fd5bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,783 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f0028c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9cbeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,785 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab34882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,786 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe0ded5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3012edbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda3efb342d8,45087,-1 for getting cluster id 2024-11-16T04:26:33,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T04:26:33,788 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8b208c1b-cf16-46a9-aa71-ed819631aff1' 2024-11-16T04:26:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T04:26:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8b208c1b-cf16-46a9-aa71-ed819631aff1" 2024-11-16T04:26:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c8f8bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda3efb342d8,45087,-1] 2024-11-16T04:26:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T04:26:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:33,789 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T04:26:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2be45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T04:26:33,790 DEBUG [Thread-2412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,790 DEBUG [Thread-2404 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,790 DEBUG [Thread-2399 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,790 DEBUG [Thread-2401 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,790 DEBUG [Thread-2410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,791 DEBUG [Thread-2395 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,791 DEBUG [Thread-2393 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,791 DEBUG [Thread-2406 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,792 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:33,792 DEBUG [Thread-2397 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,794 DEBUG [Thread-2408 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T04:26:33,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-16T04:26:33,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,795 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda3efb342d8,45455,1731731034087, seqNum=-1] 2024-11-16T04:26:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T04:26:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T04:26:33,796 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:33,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T04:26:33,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5] 2024-11-16T04:26:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:33,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:33,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:26:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731253827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731253828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731253833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731253833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731253833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c7276a352a0742a29f09a68f15d26ead_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731193804/Put/seqid=0 2024-11-16T04:26:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742362_1538 (size=17034) 2024-11-16T04:26:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T04:26:33,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731253944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731253944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731253944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731253944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731253944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:33,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:33,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:33,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:33,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T04:26:34,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731254153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731254153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731254153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731254153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731254153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,248 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:34,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c7276a352a0742a29f09a68f15d26ead_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c7276a352a0742a29f09a68f15d26ead_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:34,251 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d1397886ab84cceb60cdea2ce480707, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:34,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d1397886ab84cceb60cdea2ce480707 is 175, key is test_row_0/A:col10/1731731193804/Put/seqid=0 2024-11-16T04:26:34,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742363_1539 (size=48139) 2024-11-16T04:26:34,255 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d1397886ab84cceb60cdea2ce480707 2024-11-16T04:26:34,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/7c59c13cad8341c1bfef8ec6fe16e012 is 50, key is test_row_0/B:col10/1731731193804/Put/seqid=0 2024-11-16T04:26:34,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742364_1540 (size=12001) 2024-11-16T04:26:34,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/7c59c13cad8341c1bfef8ec6fe16e012 2024-11-16T04:26:34,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/f8e84ed478b44adc9c12d57e03ad8ebb is 50, key is test_row_0/C:col10/1731731193804/Put/seqid=0 2024-11-16T04:26:34,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742365_1541 (size=12001) 2024-11-16T04:26:34,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T04:26:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731254464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731254464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731254464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731254464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731254464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:34,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:34,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:34,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:34,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:34,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/f8e84ed478b44adc9c12d57e03ad8ebb 2024-11-16T04:26:34,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d1397886ab84cceb60cdea2ce480707 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707 2024-11-16T04:26:34,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707, entries=250, sequenceid=17, filesize=47.0 K 2024-11-16T04:26:34,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/7c59c13cad8341c1bfef8ec6fe16e012 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/7c59c13cad8341c1bfef8ec6fe16e012 2024-11-16T04:26:34,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/7c59c13cad8341c1bfef8ec6fe16e012, entries=150, sequenceid=17, filesize=11.7 K 2024-11-16T04:26:34,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/f8e84ed478b44adc9c12d57e03ad8ebb as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f8e84ed478b44adc9c12d57e03ad8ebb 2024-11-16T04:26:34,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f8e84ed478b44adc9c12d57e03ad8ebb, entries=150, sequenceid=17, filesize=11.7 K 2024-11-16T04:26:34,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for c16a25941e3d02bc3deccfcca062c9f6 in 923ms, sequenceid=17, compaction requested=false 2024-11-16T04:26:34,727 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T04:26:34,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:34,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-16T04:26:34,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:34,864 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:26:34,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162bf08549104840139cf4c03c2da41c43_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731193831/Put/seqid=0 2024-11-16T04:26:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742366_1542 (size=12154) 2024-11-16T04:26:34,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:34,874 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162bf08549104840139cf4c03c2da41c43_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162bf08549104840139cf4c03c2da41c43_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:34,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8052fc76f019401690df28905e672471, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:34,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8052fc76f019401690df28905e672471 is 175, key is test_row_0/A:col10/1731731193831/Put/seqid=0 2024-11-16T04:26:34,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742367_1543 (size=30955) 2024-11-16T04:26:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T04:26:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:34,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731254977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731254978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731254979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731254980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:34,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731254981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:34,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731255083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731255083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,086 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731255094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731255094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731255094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,278 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8052fc76f019401690df28905e672471 2024-11-16T04:26:35,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a78d38d6e65c4609a54a2734d177c2be is 50, key is test_row_0/B:col10/1731731193831/Put/seqid=0 2024-11-16T04:26:35,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742368_1544 (size=12001) 2024-11-16T04:26:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731255293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731255293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731255303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731255303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731255304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,348 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T04:26:35,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731255603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731255603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731255613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731255613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:35,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731255613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:35,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:35,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a78d38d6e65c4609a54a2734d177c2be 2024-11-16T04:26:35,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/eb347b2634e64612ac970d74042d9a59 is 50, key is test_row_0/C:col10/1731731193831/Put/seqid=0 2024-11-16T04:26:35,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742369_1545 (size=12001) 2024-11-16T04:26:35,696 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/eb347b2634e64612ac970d74042d9a59 2024-11-16T04:26:35,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8052fc76f019401690df28905e672471 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471 2024-11-16T04:26:35,702 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471, entries=150, sequenceid=41, filesize=30.2 K 2024-11-16T04:26:35,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a78d38d6e65c4609a54a2734d177c2be as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a78d38d6e65c4609a54a2734d177c2be 2024-11-16T04:26:35,705 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a78d38d6e65c4609a54a2734d177c2be, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T04:26:35,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/eb347b2634e64612ac970d74042d9a59 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/eb347b2634e64612ac970d74042d9a59 2024-11-16T04:26:35,708 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/eb347b2634e64612ac970d74042d9a59, entries=150, sequenceid=41, filesize=11.7 K 2024-11-16T04:26:35,709 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c16a25941e3d02bc3deccfcca062c9f6 in 846ms, sequenceid=41, compaction requested=false 2024-11-16T04:26:35,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:35,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:35,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-16T04:26:35,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-16T04:26:35,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-16T04:26:35,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9140 sec 2024-11-16T04:26:35,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.9200 sec 2024-11-16T04:26:35,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-16T04:26:35,944 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:35,944 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:35,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-16T04:26:35,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T04:26:35,946 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:35,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T04:26:36,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-16T04:26:36,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:36,099 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:26:36,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111624f0dc0c68ab4ebc873748f877f0bbce_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731194979/Put/seqid=0 2024-11-16T04:26:36,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742370_1546 (size=12154) 2024-11-16T04:26:36,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:36,110 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111624f0dc0c68ab4ebc873748f877f0bbce_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111624f0dc0c68ab4ebc873748f877f0bbce_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/f80a6c01d13a4be4bfad63fb0dba79ef, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/f80a6c01d13a4be4bfad63fb0dba79ef is 175, key is test_row_0/A:col10/1731731194979/Put/seqid=0 2024-11-16T04:26:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742371_1547 (size=30955) 2024-11-16T04:26:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:36,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731256134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731256135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731256139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731256139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731256140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731256244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731256244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731256253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731256253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731256253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T04:26:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731256454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731256454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731256463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731256463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731256463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,515 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/f80a6c01d13a4be4bfad63fb0dba79ef 2024-11-16T04:26:36,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/217f1a4c9db14473be205ed67e5f96a0 is 50, key is test_row_0/B:col10/1731731194979/Put/seqid=0 2024-11-16T04:26:36,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742372_1548 (size=12001) 2024-11-16T04:26:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T04:26:36,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731256764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731256764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731256773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731256773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731256773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:36,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:36,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/217f1a4c9db14473be205ed67e5f96a0 2024-11-16T04:26:36,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/fa01a09775934c8a883055f7e8489e8d is 50, key is test_row_0/C:col10/1731731194979/Put/seqid=0 2024-11-16T04:26:36,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742373_1549 (size=12001) 2024-11-16T04:26:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T04:26:37,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731257274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:37,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731257274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:37,274 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:37,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731257283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:37,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731257283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:37,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731257283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:37,332 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/fa01a09775934c8a883055f7e8489e8d 2024-11-16T04:26:37,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/f80a6c01d13a4be4bfad63fb0dba79ef as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef 2024-11-16T04:26:37,338 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef, entries=150, sequenceid=53, filesize=30.2 K 2024-11-16T04:26:37,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/217f1a4c9db14473be205ed67e5f96a0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/217f1a4c9db14473be205ed67e5f96a0 2024-11-16T04:26:37,341 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/217f1a4c9db14473be205ed67e5f96a0, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T04:26:37,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/fa01a09775934c8a883055f7e8489e8d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/fa01a09775934c8a883055f7e8489e8d 2024-11-16T04:26:37,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/fa01a09775934c8a883055f7e8489e8d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-16T04:26:37,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c16a25941e3d02bc3deccfcca062c9f6 in 1246ms, sequenceid=53, compaction requested=true 2024-11-16T04:26:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:37,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-16T04:26:37,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-16T04:26:37,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T04:26:37,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-16T04:26:37,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.4040 sec 2024-11-16T04:26:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-16T04:26:38,104 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:38,104 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-16T04:26:38,106 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T04:26:38,107 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:38,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T04:26:38,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-16T04:26:38,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:38,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:26:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162cb4205b2d874a8684ec560ef4130088_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731196134/Put/seqid=0 2024-11-16T04:26:38,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742374_1550 (size=12154) 2024-11-16T04:26:38,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:38,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731258303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731258303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,332 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:38,332 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731258328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731258328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731258328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T04:26:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731258444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731258444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731258444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731258653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731258654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731258654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:38,681 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162cb4205b2d874a8684ec560ef4130088_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162cb4205b2d874a8684ec560ef4130088_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/c37236e39d124762b1f35bc066c1c21e, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:38,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/c37236e39d124762b1f35bc066c1c21e is 175, key is test_row_0/A:col10/1731731196134/Put/seqid=0 2024-11-16T04:26:38,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742375_1551 (size=30955) 2024-11-16T04:26:38,685 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/c37236e39d124762b1f35bc066c1c21e 2024-11-16T04:26:38,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/0d7a9c866e8c49259ad787e889cb4300 is 50, key is test_row_0/B:col10/1731731196134/Put/seqid=0 2024-11-16T04:26:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742376_1552 (size=12001) 2024-11-16T04:26:38,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T04:26:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731258963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731258963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731258963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:38,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,094 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/0d7a9c866e8c49259ad787e889cb4300 2024-11-16T04:26:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/73000bfbbf0646389b6c5ec2878c5f57 is 50, key is test_row_0/C:col10/1731731196134/Put/seqid=0 2024-11-16T04:26:39,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742377_1553 (size=12001) 2024-11-16T04:26:39,103 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/73000bfbbf0646389b6c5ec2878c5f57 2024-11-16T04:26:39,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/c37236e39d124762b1f35bc066c1c21e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e 2024-11-16T04:26:39,108 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e, entries=150, sequenceid=77, filesize=30.2 K 2024-11-16T04:26:39,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/0d7a9c866e8c49259ad787e889cb4300 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/0d7a9c866e8c49259ad787e889cb4300 2024-11-16T04:26:39,111 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/0d7a9c866e8c49259ad787e889cb4300, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T04:26:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/73000bfbbf0646389b6c5ec2878c5f57 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/73000bfbbf0646389b6c5ec2878c5f57 2024-11-16T04:26:39,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/73000bfbbf0646389b6c5ec2878c5f57, entries=150, sequenceid=77, filesize=11.7 K 2024-11-16T04:26:39,114 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for c16a25941e3d02bc3deccfcca062c9f6 in 854ms, sequenceid=77, compaction requested=true 2024-11-16T04:26:39,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:39,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:39,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-16T04:26:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-16T04:26:39,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-16T04:26:39,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-16T04:26:39,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0130 sec 2024-11-16T04:26:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-16T04:26:39,243 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:39,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T04:26:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T04:26:39,245 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:39,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:39,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T04:26:39,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T04:26:39,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:39,398 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:26:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d7bf898ba5bc4fd393c98c09dfb06920_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731198302/Put/seqid=0 2024-11-16T04:26:39,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742378_1554 (size=12154) 2024-11-16T04:26:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:39,412 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d7bf898ba5bc4fd393c98c09dfb06920_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d7bf898ba5bc4fd393c98c09dfb06920_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:39,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/36ac8e5dcd99411eae0ecc7f6c2a610a, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/36ac8e5dcd99411eae0ecc7f6c2a610a is 175, key is test_row_0/A:col10/1731731198302/Put/seqid=0 2024-11-16T04:26:39,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742379_1555 (size=30955) 2024-11-16T04:26:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:39,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:39,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731259502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731259503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,507 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731259505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,509 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T04:26:39,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731259613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731259613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731259614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,816 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=90, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/36ac8e5dcd99411eae0ecc7f6c2a610a 2024-11-16T04:26:39,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/9016ca7af9e74517bff0cd26c3149b7e is 50, key is test_row_0/B:col10/1731731198302/Put/seqid=0 2024-11-16T04:26:39,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742380_1556 (size=12001) 2024-11-16T04:26:39,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731259823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731259823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:39,826 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731259823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:39,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T04:26:40,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731260134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731260134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731260134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,136 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/9016ca7af9e74517bff0cd26c3149b7e 2024-11-16T04:26:40,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/857bb4c3e8a74853bace93ed2c68f215 is 50, key is test_row_0/C:col10/1731731198302/Put/seqid=0 2024-11-16T04:26:40,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742381_1557 (size=12001) 2024-11-16T04:26:40,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731260353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731260353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,356 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:40,356 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T04:26:40,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/857bb4c3e8a74853bace93ed2c68f215 2024-11-16T04:26:40,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/36ac8e5dcd99411eae0ecc7f6c2a610a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a 2024-11-16T04:26:40,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a, entries=150, sequenceid=90, filesize=30.2 K 2024-11-16T04:26:40,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/9016ca7af9e74517bff0cd26c3149b7e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/9016ca7af9e74517bff0cd26c3149b7e 2024-11-16T04:26:40,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/9016ca7af9e74517bff0cd26c3149b7e, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T04:26:40,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/857bb4c3e8a74853bace93ed2c68f215 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/857bb4c3e8a74853bace93ed2c68f215 2024-11-16T04:26:40,645 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/857bb4c3e8a74853bace93ed2c68f215, entries=150, sequenceid=90, filesize=11.7 K 2024-11-16T04:26:40,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731260644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,646 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c16a25941e3d02bc3deccfcca062c9f6 in 1248ms, sequenceid=90, compaction requested=true 2024-11-16T04:26:40,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:40,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:40,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T04:26:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T04:26:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:40,646 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:40,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-16T04:26:40,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T04:26:40,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-16T04:26:40,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4050 sec 2024-11-16T04:26:40,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116756d7b6818954850b671d539d0be8ede_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:40,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742382_1558 (size=14594) 2024-11-16T04:26:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731260670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731260671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731260783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731260783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731260994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731260994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:40,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:41,055 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:41,058 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116756d7b6818954850b671d539d0be8ede_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116756d7b6818954850b671d539d0be8ede_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:41,059 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/9d95f4a042df40a9acf61df77710f20e, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:41,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/9d95f4a042df40a9acf61df77710f20e is 175, key is test_row_0/A:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:41,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742383_1559 (size=39549) 2024-11-16T04:26:41,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731261303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731261303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T04:26:41,393 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:41,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T04:26:41,395 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T04:26:41,395 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:41,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:41,462 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/9d95f4a042df40a9acf61df77710f20e 2024-11-16T04:26:41,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/721f405d8aa04a9eb747c9c1f5900f77 is 50, key is test_row_0/B:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:41,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742384_1560 (size=12001) 2024-11-16T04:26:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T04:26:41,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T04:26:41,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:41,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731261653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:41,656 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:41,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T04:26:41,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:41,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T04:26:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731261813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:41,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731261813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:41,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T04:26:41,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:41,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:41,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/721f405d8aa04a9eb747c9c1f5900f77 2024-11-16T04:26:41,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/18b19e7302424fe09b8b8d640691ba25 is 50, key is test_row_0/C:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742385_1561 (size=12001) 2024-11-16T04:26:41,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/18b19e7302424fe09b8b8d640691ba25 2024-11-16T04:26:41,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/9d95f4a042df40a9acf61df77710f20e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e 2024-11-16T04:26:41,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e, entries=200, sequenceid=114, filesize=38.6 K 2024-11-16T04:26:41,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/721f405d8aa04a9eb747c9c1f5900f77 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/721f405d8aa04a9eb747c9c1f5900f77 2024-11-16T04:26:41,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/721f405d8aa04a9eb747c9c1f5900f77, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T04:26:41,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/18b19e7302424fe09b8b8d640691ba25 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/18b19e7302424fe09b8b8d640691ba25 2024-11-16T04:26:41,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/18b19e7302424fe09b8b8d640691ba25, entries=150, sequenceid=114, filesize=11.7 K 2024-11-16T04:26:41,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c16a25941e3d02bc3deccfcca062c9f6 in 1256ms, sequenceid=114, compaction requested=true 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:41,902 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:41,902 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T04:26:41,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:41,904 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 211508 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T04:26:41,904 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/A is initiating minor compaction (all files) 2024-11-16T04:26:41,904 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/A in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,904 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=206.6 K 2024-11-16T04:26:41,904 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,904 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e] 2024-11-16T04:26:41,904 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T04:26:41,904 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/B is initiating minor compaction (all files) 2024-11-16T04:26:41,904 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/B in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,905 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/7c59c13cad8341c1bfef8ec6fe16e012, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a78d38d6e65c4609a54a2734d177c2be, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/217f1a4c9db14473be205ed67e5f96a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/0d7a9c866e8c49259ad787e889cb4300, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/9016ca7af9e74517bff0cd26c3149b7e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/721f405d8aa04a9eb747c9c1f5900f77] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=70.3 K 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d1397886ab84cceb60cdea2ce480707, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731731193797 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c59c13cad8341c1bfef8ec6fe16e012, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731731193802 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8052fc76f019401690df28905e672471, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731731193831 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a78d38d6e65c4609a54a2734d177c2be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731731193831 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80a6c01d13a4be4bfad63fb0dba79ef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731731194976 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 217f1a4c9db14473be205ed67e5f96a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731731194976 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c37236e39d124762b1f35bc066c1c21e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731731196134 2024-11-16T04:26:41,905 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d7a9c866e8c49259ad787e889cb4300, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731731196134 2024-11-16T04:26:41,906 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ac8e5dcd99411eae0ecc7f6c2a610a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731731198296 2024-11-16T04:26:41,906 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 9016ca7af9e74517bff0cd26c3149b7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731731198296 2024-11-16T04:26:41,906 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d95f4a042df40a9acf61df77710f20e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731731199497 2024-11-16T04:26:41,906 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 721f405d8aa04a9eb747c9c1f5900f77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731731199497 2024-11-16T04:26:41,913 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:41,914 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116117b4b9fed894f8dbc5ade33c23af517_c16a25941e3d02bc3deccfcca062c9f6 store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:41,917 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116117b4b9fed894f8dbc5ade33c23af517_c16a25941e3d02bc3deccfcca062c9f6, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:41,917 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#B#compaction#481 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:41,917 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116117b4b9fed894f8dbc5ade33c23af517_c16a25941e3d02bc3deccfcca062c9f6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:41,917 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/e7a4b1b3db3a4dc1b50944cbadcd149e is 50, key is test_row_0/B:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:41,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742386_1562 (size=4469) 2024-11-16T04:26:41,922 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#A#compaction#480 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:41,923 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/82bece34cdfe4f2f862d97bc28b6fe6b is 175, key is test_row_0/A:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742387_1563 (size=12207) 2024-11-16T04:26:41,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742388_1564 (size=31161) 2024-11-16T04:26:41,937 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/82bece34cdfe4f2f862d97bc28b6fe6b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/82bece34cdfe4f2f862d97bc28b6fe6b 2024-11-16T04:26:41,941 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/A of c16a25941e3d02bc3deccfcca062c9f6 into 82bece34cdfe4f2f862d97bc28b6fe6b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:41,941 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:41,941 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/A, priority=10, startTime=1731731201902; duration=0sec 2024-11-16T04:26:41,941 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:41,941 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:A 2024-11-16T04:26:41,941 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-16T04:26:41,942 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-16T04:26:41,942 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/C is initiating minor compaction (all files) 2024-11-16T04:26:41,942 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/C in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:41,942 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f8e84ed478b44adc9c12d57e03ad8ebb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/eb347b2634e64612ac970d74042d9a59, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/fa01a09775934c8a883055f7e8489e8d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/73000bfbbf0646389b6c5ec2878c5f57, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/857bb4c3e8a74853bace93ed2c68f215, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/18b19e7302424fe09b8b8d640691ba25] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=70.3 K 2024-11-16T04:26:41,943 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8e84ed478b44adc9c12d57e03ad8ebb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731731193802 2024-11-16T04:26:41,943 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb347b2634e64612ac970d74042d9a59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731731193831 2024-11-16T04:26:41,944 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa01a09775934c8a883055f7e8489e8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731731194976 2024-11-16T04:26:41,944 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73000bfbbf0646389b6c5ec2878c5f57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731731196134 2024-11-16T04:26:41,944 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 857bb4c3e8a74853bace93ed2c68f215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731731198296 2024-11-16T04:26:41,944 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18b19e7302424fe09b8b8d640691ba25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731731199497 2024-11-16T04:26:41,954 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#C#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:41,955 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/610b6ff06d66441eb166cfe68a0180b7 is 50, key is test_row_0/C:col10/1731731200645/Put/seqid=0 2024-11-16T04:26:41,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742389_1565 (size=12207) 2024-11-16T04:26:41,962 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/610b6ff06d66441eb166cfe68a0180b7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/610b6ff06d66441eb166cfe68a0180b7 2024-11-16T04:26:41,966 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/C of c16a25941e3d02bc3deccfcca062c9f6 into 610b6ff06d66441eb166cfe68a0180b7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:41,966 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:41,966 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/C, priority=10, startTime=1731731201902; duration=0sec 2024-11-16T04:26:41,966 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:41,966 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:C 2024-11-16T04:26:42,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T04:26:42,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:42,005 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-16T04:26:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb9fe2819e2f41e4b0ab0900027659d5_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731200668/Put/seqid=0 2024-11-16T04:26:42,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742390_1566 (size=12154) 2024-11-16T04:26:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T04:26:42,329 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/e7a4b1b3db3a4dc1b50944cbadcd149e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e7a4b1b3db3a4dc1b50944cbadcd149e 2024-11-16T04:26:42,332 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/B of c16a25941e3d02bc3deccfcca062c9f6 into e7a4b1b3db3a4dc1b50944cbadcd149e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:42,332 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:42,332 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/B, priority=10, startTime=1731731201902; duration=0sec 2024-11-16T04:26:42,332 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:42,332 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:B 2024-11-16T04:26:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:42,419 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116eb9fe2819e2f41e4b0ab0900027659d5_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb9fe2819e2f41e4b0ab0900027659d5_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:42,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/efd05243140c4ddeada27df360ffcab1, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:42,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/efd05243140c4ddeada27df360ffcab1 is 175, key is test_row_0/A:col10/1731731200668/Put/seqid=0 2024-11-16T04:26:42,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742391_1567 (size=30955) 2024-11-16T04:26:42,423 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/efd05243140c4ddeada27df360ffcab1 2024-11-16T04:26:42,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4 is 50, key is test_row_0/B:col10/1731731200668/Put/seqid=0 2024-11-16T04:26:42,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742392_1568 (size=12001) 2024-11-16T04:26:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T04:26:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:42,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:42,831 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4 2024-11-16T04:26:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/2a205aa7218841e99efb779c2359cc83 is 50, key is test_row_0/C:col10/1731731200668/Put/seqid=0 2024-11-16T04:26:42,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742393_1569 (size=12001) 2024-11-16T04:26:42,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731262883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:42,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731262885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,888 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:42,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731262994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:42,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731262994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:42,996 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731263204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731263204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/2a205aa7218841e99efb779c2359cc83 2024-11-16T04:26:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/efd05243140c4ddeada27df360ffcab1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1 2024-11-16T04:26:43,248 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1, entries=150, sequenceid=128, filesize=30.2 K 2024-11-16T04:26:43,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4 2024-11-16T04:26:43,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T04:26:43,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/2a205aa7218841e99efb779c2359cc83 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2a205aa7218841e99efb779c2359cc83 2024-11-16T04:26:43,254 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2a205aa7218841e99efb779c2359cc83, entries=150, sequenceid=128, filesize=11.7 K 2024-11-16T04:26:43,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c16a25941e3d02bc3deccfcca062c9f6 in 1250ms, sequenceid=128, compaction requested=false 2024-11-16T04:26:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T04:26:43,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T04:26:43,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T04:26:43,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8610 sec 2024-11-16T04:26:43,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8630 sec 2024-11-16T04:26:43,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:43,516 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:43,516 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-16T04:26:43,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116296a6546d8904915aed207c465e11c2a_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:43,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742394_1570 (size=17284) 2024-11-16T04:26:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731263530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731263533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T04:26:43,544 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:43,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T04:26:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T04:26:43,545 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:43,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:43,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731263643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731263643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,647 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T04:26:43,668 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T04:26:43,668 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T04:26:43,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731263683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,685 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_2' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:43,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:43,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:43,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:43,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:43,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731263853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731263853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T04:26:43,926 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:43,928 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116296a6546d8904915aed207c465e11c2a_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116296a6546d8904915aed207c465e11c2a_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:43,929 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/3dd940b988044e20b74a7a7b3ecfb268, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:43,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/3dd940b988044e20b74a7a7b3ecfb268 is 175, key is test_row_0/A:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:43,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742395_1571 (size=48389) 2024-11-16T04:26:43,932 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/3dd940b988044e20b74a7a7b3ecfb268 2024-11-16T04:26:43,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/d8814b8a4256434ea197d88aaf1d483a is 50, key is test_row_0/B:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:43,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742396_1572 (size=12151) 2024-11-16T04:26:44,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:44,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731264164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:44,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731264164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,166 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:44,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T04:26:44,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/d8814b8a4256434ea197d88aaf1d483a 2024-11-16T04:26:44,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/b69c9e98deab4929a10681f4e80bab48 is 50, key is test_row_0/C:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:44,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742397_1573 (size=12151) 2024-11-16T04:26:44,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48118 deadline: 1731731264373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:44,376 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:44,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48096 deadline: 1731731264393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:44,394 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:44,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731264673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:44,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731264673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:44,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T04:26:44,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/b69c9e98deab4929a10681f4e80bab48 2024-11-16T04:26:44,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/3dd940b988044e20b74a7a7b3ecfb268 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268 2024-11-16T04:26:44,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268, entries=250, sequenceid=155, filesize=47.3 K 2024-11-16T04:26:44,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/d8814b8a4256434ea197d88aaf1d483a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/d8814b8a4256434ea197d88aaf1d483a 2024-11-16T04:26:44,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/d8814b8a4256434ea197d88aaf1d483a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T04:26:44,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/b69c9e98deab4929a10681f4e80bab48 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/b69c9e98deab4929a10681f4e80bab48 2024-11-16T04:26:44,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/b69c9e98deab4929a10681f4e80bab48, entries=150, sequenceid=155, filesize=11.9 K 2024-11-16T04:26:44,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for c16a25941e3d02bc3deccfcca062c9f6 in 1245ms, sequenceid=155, compaction requested=true 2024-11-16T04:26:44,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:44,762 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:44,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:44,762 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:44,762 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:44,762 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/B is initiating minor compaction (all files) 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/A is initiating minor compaction (all files) 2024-11-16T04:26:44,763 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/A in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,763 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/B in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,763 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/82bece34cdfe4f2f862d97bc28b6fe6b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=107.9 K 2024-11-16T04:26:44,763 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e7a4b1b3db3a4dc1b50944cbadcd149e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/d8814b8a4256434ea197d88aaf1d483a] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=35.5 K 2024-11-16T04:26:44,763 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/82bece34cdfe4f2f862d97bc28b6fe6b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268] 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a4b1b3db3a4dc1b50944cbadcd149e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731731199497 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82bece34cdfe4f2f862d97bc28b6fe6b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731731199497 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa343ebd57b4f2dbe33f6f86ad3f1c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731731200659 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd05243140c4ddeada27df360ffcab1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731731200659 2024-11-16T04:26:44,763 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting d8814b8a4256434ea197d88aaf1d483a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731202880 2024-11-16T04:26:44,764 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd940b988044e20b74a7a7b3ecfb268, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731202880 2024-11-16T04:26:44,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T04:26:44,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,766 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-16T04:26:44,771 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:44,772 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/3693f45be6db488fb40b424a7e544680 is 50, key is test_row_0/B:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c2f3cdab60df40aeb71e145649f38156_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731203532/Put/seqid=0 2024-11-16T04:26:44,773 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:44,777 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164fda690ef31c4d7fb2a1da8912e9e8f4_c16a25941e3d02bc3deccfcca062c9f6 store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:44,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742398_1574 (size=12304) 2024-11-16T04:26:44,779 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164fda690ef31c4d7fb2a1da8912e9e8f4_c16a25941e3d02bc3deccfcca062c9f6, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:44,779 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164fda690ef31c4d7fb2a1da8912e9e8f4_c16a25941e3d02bc3deccfcca062c9f6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:44,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742399_1575 (size=12459) 2024-11-16T04:26:44,785 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/3693f45be6db488fb40b424a7e544680 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/3693f45be6db488fb40b424a7e544680 2024-11-16T04:26:44,789 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/B of c16a25941e3d02bc3deccfcca062c9f6 into 3693f45be6db488fb40b424a7e544680(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:44,789 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:44,789 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/B, priority=13, startTime=1731731204762; duration=0sec 2024-11-16T04:26:44,789 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:44,789 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:B 2024-11-16T04:26:44,789 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:44,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742400_1576 (size=4469) 2024-11-16T04:26:44,790 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:44,790 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/C is initiating minor compaction (all files) 2024-11-16T04:26:44,790 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/C in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:44,790 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/610b6ff06d66441eb166cfe68a0180b7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2a205aa7218841e99efb779c2359cc83, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/b69c9e98deab4929a10681f4e80bab48] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=35.5 K 2024-11-16T04:26:44,790 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 610b6ff06d66441eb166cfe68a0180b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731731199497 2024-11-16T04:26:44,791 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a205aa7218841e99efb779c2359cc83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731731200659 2024-11-16T04:26:44,791 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting b69c9e98deab4929a10681f4e80bab48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731202880 2024-11-16T04:26:44,796 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:44,797 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/13837645178143a1ae88505941807ad5 is 50, key is test_row_0/C:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:44,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742401_1577 (size=12459) 2024-11-16T04:26:44,804 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/13837645178143a1ae88505941807ad5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/13837645178143a1ae88505941807ad5 2024-11-16T04:26:44,807 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/C of c16a25941e3d02bc3deccfcca062c9f6 into 13837645178143a1ae88505941807ad5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:44,807 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:44,807 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/C, priority=13, startTime=1731731204762; duration=0sec 2024-11-16T04:26:44,807 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:44,807 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:C 2024-11-16T04:26:45,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,181 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c2f3cdab60df40aeb71e145649f38156_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c2f3cdab60df40aeb71e145649f38156_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:45,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/7796b8aebb6e4fc39271b115a3a2df27, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:45,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/7796b8aebb6e4fc39271b115a3a2df27 is 175, key is test_row_0/A:col10/1731731203532/Put/seqid=0 2024-11-16T04:26:45,190 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#A#compaction#491 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:45,191 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/9cd278312b1d467bb23241036e492142 is 175, key is test_row_0/A:col10/1731731202880/Put/seqid=0 2024-11-16T04:26:45,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742402_1578 (size=31105) 2024-11-16T04:26:45,194 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/7796b8aebb6e4fc39271b115a3a2df27 2024-11-16T04:26:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a32854f9f8794486ad22f79b6daf1e0e is 50, key is test_row_0/B:col10/1731731203532/Put/seqid=0 2024-11-16T04:26:45,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742403_1579 (size=31413) 2024-11-16T04:26:45,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742404_1580 (size=12151) 2024-11-16T04:26:45,219 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/9cd278312b1d467bb23241036e492142 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9cd278312b1d467bb23241036e492142 2024-11-16T04:26:45,220 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a32854f9f8794486ad22f79b6daf1e0e 2024-11-16T04:26:45,225 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/A of c16a25941e3d02bc3deccfcca062c9f6 into 9cd278312b1d467bb23241036e492142(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:45,225 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:45,225 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/A, priority=13, startTime=1731731204762; duration=0sec 2024-11-16T04:26:45,225 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:45,225 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:A 2024-11-16T04:26:45,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/cb442f0ec9fa48cb9b8f710bd43049ee is 50, key is test_row_0/C:col10/1731731203532/Put/seqid=0 2024-11-16T04:26:45,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742405_1581 (size=12151) 2024-11-16T04:26:45,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/cb442f0ec9fa48cb9b8f710bd43049ee 2024-11-16T04:26:45,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/7796b8aebb6e4fc39271b115a3a2df27 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27 2024-11-16T04:26:45,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27, entries=150, sequenceid=165, filesize=30.4 K 2024-11-16T04:26:45,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a32854f9f8794486ad22f79b6daf1e0e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a32854f9f8794486ad22f79b6daf1e0e 2024-11-16T04:26:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,643 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a32854f9f8794486ad22f79b6daf1e0e, entries=150, sequenceid=165, filesize=11.9 K 2024-11-16T04:26:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/cb442f0ec9fa48cb9b8f710bd43049ee as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cb442f0ec9fa48cb9b8f710bd43049ee 2024-11-16T04:26:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cb442f0ec9fa48cb9b8f710bd43049ee, entries=150, sequenceid=165, filesize=11.9 K 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for c16a25941e3d02bc3deccfcca062c9f6 in 882ms, sequenceid=165, compaction requested=false 2024-11-16T04:26:45,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:45,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:45,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1020 sec 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1070 sec 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T04:26:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,704 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:45,704 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T04:26:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,706 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T04:26:45,707 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,725 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:26:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:45,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111681b29a1cd57d4dcea3f02dee48ecc505_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:45,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742407_1583 (size=24758) 2024-11-16T04:26:45,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731265778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:45,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731265781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T04:26:45,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:45,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:45,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:45,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:45,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:45,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:45,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731265893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:45,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731265893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T04:26:46,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731266104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731266104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,139 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:46,142 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111681b29a1cd57d4dcea3f02dee48ecc505_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111681b29a1cd57d4dcea3f02dee48ecc505_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:46,143 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/da81dbd5131c4da8ad5ce176c2012142, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:46,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/da81dbd5131c4da8ad5ce176c2012142 is 175, key is test_row_0/A:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:46,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742406_1582 (size=74395) 2024-11-16T04:26:46,149 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/da81dbd5131c4da8ad5ce176c2012142 2024-11-16T04:26:46,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/14f7bee99ef74608879aaf219bd9052b is 50, key is test_row_0/B:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:46,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742408_1584 (size=12151) 2024-11-16T04:26:46,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T04:26:46,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731266413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:46,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731266413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,416 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/14f7bee99ef74608879aaf219bd9052b 2024-11-16T04:26:46,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/bcc03dd3f67444eb9a374c045c60a2c4 is 50, key is test_row_0/C:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:46,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742409_1585 (size=12151) 2024-11-16T04:26:46,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T04:26:46,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731266923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731266923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:46,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:46,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:46,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/bcc03dd3f67444eb9a374c045c60a2c4 2024-11-16T04:26:46,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/da81dbd5131c4da8ad5ce176c2012142 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142 2024-11-16T04:26:46,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142, entries=400, sequenceid=179, filesize=72.7 K 2024-11-16T04:26:46,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/14f7bee99ef74608879aaf219bd9052b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/14f7bee99ef74608879aaf219bd9052b 2024-11-16T04:26:46,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/14f7bee99ef74608879aaf219bd9052b, entries=150, sequenceid=179, filesize=11.9 K 2024-11-16T04:26:46,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/bcc03dd3f67444eb9a374c045c60a2c4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/bcc03dd3f67444eb9a374c045c60a2c4 2024-11-16T04:26:46,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/bcc03dd3f67444eb9a374c045c60a2c4, entries=150, sequenceid=179, filesize=11.9 K 2024-11-16T04:26:46,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c16a25941e3d02bc3deccfcca062c9f6 in 1255ms, sequenceid=179, compaction requested=true 2024-11-16T04:26:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:46,980 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:46,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:46,981 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:46,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:46,981 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:46,981 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136913 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:46,981 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/B is initiating minor compaction (all files) 2024-11-16T04:26:46,981 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/A is initiating minor compaction (all files) 2024-11-16T04:26:46,981 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/B in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,981 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/A in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,982 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9cd278312b1d467bb23241036e492142, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=133.7 K 2024-11-16T04:26:46,982 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/3693f45be6db488fb40b424a7e544680, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a32854f9f8794486ad22f79b6daf1e0e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/14f7bee99ef74608879aaf219bd9052b] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=35.9 K 2024-11-16T04:26:46,982 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:46,982 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9cd278312b1d467bb23241036e492142, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142] 2024-11-16T04:26:46,982 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 3693f45be6db488fb40b424a7e544680, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731202880 2024-11-16T04:26:46,982 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cd278312b1d467bb23241036e492142, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731202880 2024-11-16T04:26:46,982 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7796b8aebb6e4fc39271b115a3a2df27, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731731203524 2024-11-16T04:26:46,982 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a32854f9f8794486ad22f79b6daf1e0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731731203524 2024-11-16T04:26:46,983 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 14f7bee99ef74608879aaf219bd9052b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731731205698 2024-11-16T04:26:46,983 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting da81dbd5131c4da8ad5ce176c2012142, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731731205698 2024-11-16T04:26:46,993 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:46,994 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:46,994 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116c94963e73b024241b2ac68ecbf1c8073_c16a25941e3d02bc3deccfcca062c9f6 store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:46,995 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/439e83c546114ca7a8b63f9a0421dd93 is 50, key is test_row_0/B:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:46,997 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116c94963e73b024241b2ac68ecbf1c8073_c16a25941e3d02bc3deccfcca062c9f6, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:46,997 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c94963e73b024241b2ac68ecbf1c8073_c16a25941e3d02bc3deccfcca062c9f6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742410_1586 (size=12561) 2024-11-16T04:26:47,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742411_1587 (size=4469) 2024-11-16T04:26:47,016 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/439e83c546114ca7a8b63f9a0421dd93 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/439e83c546114ca7a8b63f9a0421dd93 2024-11-16T04:26:47,019 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/B of c16a25941e3d02bc3deccfcca062c9f6 into 439e83c546114ca7a8b63f9a0421dd93(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:47,019 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:47,019 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/B, priority=13, startTime=1731731206980; duration=0sec 2024-11-16T04:26:47,019 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:47,019 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:B 2024-11-16T04:26:47,019 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:47,020 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:47,020 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/C is initiating minor compaction (all files) 2024-11-16T04:26:47,020 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/C in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:47,020 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/13837645178143a1ae88505941807ad5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cb442f0ec9fa48cb9b8f710bd43049ee, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/bcc03dd3f67444eb9a374c045c60a2c4] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=35.9 K 2024-11-16T04:26:47,021 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 13837645178143a1ae88505941807ad5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731731202880 2024-11-16T04:26:47,021 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting cb442f0ec9fa48cb9b8f710bd43049ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731731203524 2024-11-16T04:26:47,021 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc03dd3f67444eb9a374c045c60a2c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731731205698 2024-11-16T04:26:47,028 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#C#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:47,028 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/682e7f784c4b4b419ae0847a36cd4c27 is 50, key is test_row_0/C:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:47,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742412_1588 (size=12561) 2024-11-16T04:26:47,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T04:26:47,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:47,079 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:26:47,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111602a3756c3fb34d409f890f86a5769f9a_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731205777/Put/seqid=0 2024-11-16T04:26:47,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742413_1589 (size=12304) 2024-11-16T04:26:47,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,096 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111602a3756c3fb34d409f890f86a5769f9a_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602a3756c3fb34d409f890f86a5769f9a_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:47,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d36deb3ef7034707a61a2d68b512b851, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:47,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d36deb3ef7034707a61a2d68b512b851 is 175, key is test_row_0/A:col10/1731731205777/Put/seqid=0 2024-11-16T04:26:47,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742414_1590 (size=31105) 2024-11-16T04:26:47,100 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d36deb3ef7034707a61a2d68b512b851 2024-11-16T04:26:47,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a1717ac72a0343b8a3b4b73902510e48 is 50, key is test_row_0/B:col10/1731731205777/Put/seqid=0 2024-11-16T04:26:47,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742415_1591 (size=12151) 2024-11-16T04:26:47,109 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a1717ac72a0343b8a3b4b73902510e48 2024-11-16T04:26:47,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/44d3948b600e4c2c95516fbe7be63c7c is 50, key is test_row_0/C:col10/1731731205777/Put/seqid=0 2024-11-16T04:26:47,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742416_1592 (size=12151) 2024-11-16T04:26:47,406 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#A#compaction#499 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:47,407 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/09854141b8c0472a8233439af3753611 is 175, key is test_row_0/A:col10/1731731205723/Put/seqid=0 2024-11-16T04:26:47,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742417_1593 (size=31515) 2024-11-16T04:26:47,414 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/09854141b8c0472a8233439af3753611 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/09854141b8c0472a8233439af3753611 2024-11-16T04:26:47,418 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/A of c16a25941e3d02bc3deccfcca062c9f6 into 09854141b8c0472a8233439af3753611(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:47,418 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:47,418 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/A, priority=13, startTime=1731731206980; duration=0sec 2024-11-16T04:26:47,418 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:47,418 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:A 2024-11-16T04:26:47,445 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/682e7f784c4b4b419ae0847a36cd4c27 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/682e7f784c4b4b419ae0847a36cd4c27 2024-11-16T04:26:47,449 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/C of c16a25941e3d02bc3deccfcca062c9f6 into 682e7f784c4b4b419ae0847a36cd4c27(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:47,449 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:47,449 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/C, priority=13, startTime=1731731206981; duration=0sec 2024-11-16T04:26:47,449 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:47,449 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:C 2024-11-16T04:26:47,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/44d3948b600e4c2c95516fbe7be63c7c 2024-11-16T04:26:47,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d36deb3ef7034707a61a2d68b512b851 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851 2024-11-16T04:26:47,524 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851, entries=150, sequenceid=205, filesize=30.4 K 2024-11-16T04:26:47,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a1717ac72a0343b8a3b4b73902510e48 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a1717ac72a0343b8a3b4b73902510e48 2024-11-16T04:26:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,527 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a1717ac72a0343b8a3b4b73902510e48, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T04:26:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/44d3948b600e4c2c95516fbe7be63c7c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/44d3948b600e4c2c95516fbe7be63c7c 2024-11-16T04:26:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/44d3948b600e4c2c95516fbe7be63c7c, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T04:26:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for c16a25941e3d02bc3deccfcca062c9f6 in 453ms, sequenceid=205, compaction requested=false 2024-11-16T04:26:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T04:26:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T04:26:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8270 sec 2024-11-16T04:26:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8320 sec 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:47,809 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:47,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-16T04:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f2b19a2d815a47a4b16023d073a4bef6_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742418_1594 (size=17284) 2024-11-16T04:26:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T04:26:47,853 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:47,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T04:26:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:47,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:47,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:47,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:47,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731267924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:47,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:47,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:47,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731267934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:47,936 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:47,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:47,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:47,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731267943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:47,946 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_1' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:47,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:48,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:48,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731268033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:48,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:48,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:48,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:48,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f2b19a2d815a47a4b16023d073a4bef6_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f2b19a2d815a47a4b16023d073a4bef6_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:48,226 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d0b97b7de4a4493aa3ba8d377b1dbe5e, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:48,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d0b97b7de4a4493aa3ba8d377b1dbe5e is 175, key is test_row_0/A:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:48,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742419_1595 (size=48389) 2024-11-16T04:26:48,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731268243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:48,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:48,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:48,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731268554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:48,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d0b97b7de4a4493aa3ba8d377b1dbe5e 2024-11-16T04:26:48,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/cb6755a7802549658170d3dd1b64f383 is 50, key is test_row_0/B:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:48,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742420_1596 (size=12151) 2024-11-16T04:26:48,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:48,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:49,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731269063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:49,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/cb6755a7802549658170d3dd1b64f383 2024-11-16T04:26:49,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/2c3de6a81a7d4004b21a842e3c090dce is 50, key is test_row_0/C:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:49,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742421_1597 (size=12151) 2024-11-16T04:26:49,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:49,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:49,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:49,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/2c3de6a81a7d4004b21a842e3c090dce 2024-11-16T04:26:49,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d0b97b7de4a4493aa3ba8d377b1dbe5e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e 2024-11-16T04:26:49,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e, entries=250, sequenceid=218, filesize=47.3 K 2024-11-16T04:26:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/cb6755a7802549658170d3dd1b64f383 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/cb6755a7802549658170d3dd1b64f383 2024-11-16T04:26:49,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/cb6755a7802549658170d3dd1b64f383, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T04:26:49,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/2c3de6a81a7d4004b21a842e3c090dce as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2c3de6a81a7d4004b21a842e3c090dce 2024-11-16T04:26:49,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2c3de6a81a7d4004b21a842e3c090dce, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T04:26:49,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c16a25941e3d02bc3deccfcca062c9f6 in 1705ms, sequenceid=218, compaction requested=true 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:49,515 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:49,515 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:49,515 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:49,516 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/A is initiating minor compaction (all files) 2024-11-16T04:26:49,516 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/A in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,516 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/09854141b8c0472a8233439af3753611, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=108.4 K 2024-11-16T04:26:49,516 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,516 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/09854141b8c0472a8233439af3753611, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e] 2024-11-16T04:26:49,516 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:49,516 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09854141b8c0472a8233439af3753611, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731731205698 2024-11-16T04:26:49,516 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/B is initiating minor compaction (all files) 2024-11-16T04:26:49,516 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/B in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,516 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/439e83c546114ca7a8b63f9a0421dd93, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a1717ac72a0343b8a3b4b73902510e48, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/cb6755a7802549658170d3dd1b64f383] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=36.0 K 2024-11-16T04:26:49,516 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d36deb3ef7034707a61a2d68b512b851, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731731205774 2024-11-16T04:26:49,517 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0b97b7de4a4493aa3ba8d377b1dbe5e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731731207762 2024-11-16T04:26:49,517 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 439e83c546114ca7a8b63f9a0421dd93, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731731205698 2024-11-16T04:26:49,517 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a1717ac72a0343b8a3b4b73902510e48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731731205774 2024-11-16T04:26:49,517 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6755a7802549658170d3dd1b64f383, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731731207789 2024-11-16T04:26:49,529 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:49,530 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111613da34a46b0a4bf2a222adcec1a42e87_c16a25941e3d02bc3deccfcca062c9f6 store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:49,531 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#B#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:49,531 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/15a06e6abee0412f8c78730945e6f368 is 50, key is test_row_0/B:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:49,532 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111613da34a46b0a4bf2a222adcec1a42e87_c16a25941e3d02bc3deccfcca062c9f6, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:49,532 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111613da34a46b0a4bf2a222adcec1a42e87_c16a25941e3d02bc3deccfcca062c9f6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:49,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742423_1599 (size=4469) 2024-11-16T04:26:49,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742422_1598 (size=12663) 2024-11-16T04:26:49,537 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#A#compaction#507 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:49,537 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/c823aaf944b2418aac8a880ff9e54938 is 175, key is test_row_0/A:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742424_1600 (size=31617) 2024-11-16T04:26:49,546 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/c823aaf944b2418aac8a880ff9e54938 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c823aaf944b2418aac8a880ff9e54938 2024-11-16T04:26:49,550 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/A of c16a25941e3d02bc3deccfcca062c9f6 into c823aaf944b2418aac8a880ff9e54938(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:49,550 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:49,550 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/A, priority=13, startTime=1731731209515; duration=0sec 2024-11-16T04:26:49,550 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:49,550 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:A 2024-11-16T04:26:49,550 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:49,550 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:49,551 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/C is initiating minor compaction (all files) 2024-11-16T04:26:49,551 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/C in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,551 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/682e7f784c4b4b419ae0847a36cd4c27, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/44d3948b600e4c2c95516fbe7be63c7c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2c3de6a81a7d4004b21a842e3c090dce] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=36.0 K 2024-11-16T04:26:49,552 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682e7f784c4b4b419ae0847a36cd4c27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731731205698 2024-11-16T04:26:49,552 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d3948b600e4c2c95516fbe7be63c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731731205774 2024-11-16T04:26:49,552 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c3de6a81a7d4004b21a842e3c090dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731731207789 2024-11-16T04:26:49,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T04:26:49,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:49,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-16T04:26:49,558 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:49,559 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/adf7359bd81540acbaef68ac710f7935 is 50, key is test_row_0/C:col10/1731731207808/Put/seqid=0 2024-11-16T04:26:49,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116959bdbee86ca4a3ba10250e51cf35773_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731207890/Put/seqid=0 2024-11-16T04:26:49,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742425_1601 (size=12663) 2024-11-16T04:26:49,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742426_1602 (size=12304) 2024-11-16T04:26:49,571 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/adf7359bd81540acbaef68ac710f7935 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/adf7359bd81540acbaef68ac710f7935 2024-11-16T04:26:49,575 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/C of c16a25941e3d02bc3deccfcca062c9f6 into adf7359bd81540acbaef68ac710f7935(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:49,575 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:49,575 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/C, priority=13, startTime=1731731209515; duration=0sec 2024-11-16T04:26:49,575 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:49,575 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:C 2024-11-16T04:26:49,939 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/15a06e6abee0412f8c78730945e6f368 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/15a06e6abee0412f8c78730945e6f368 2024-11-16T04:26:49,943 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/B of c16a25941e3d02bc3deccfcca062c9f6 into 15a06e6abee0412f8c78730945e6f368(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:49,943 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:49,943 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/B, priority=13, startTime=1731731209515; duration=0sec 2024-11-16T04:26:49,943 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:49,943 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:B 2024-11-16T04:26:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:49,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:49,971 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116959bdbee86ca4a3ba10250e51cf35773_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116959bdbee86ca4a3ba10250e51cf35773_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:49,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/995cc21c45e541058c087ad89860a868, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:49,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/995cc21c45e541058c087ad89860a868 is 175, key is test_row_0/A:col10/1731731207890/Put/seqid=0 2024-11-16T04:26:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731269970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:49,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742427_1603 (size=31105) 2024-11-16T04:26:49,975 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/995cc21c45e541058c087ad89860a868 2024-11-16T04:26:49,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731269972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:49,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a0a48b28bc9849b78058ba2e23c5acfc is 50, key is test_row_0/B:col10/1731731207890/Put/seqid=0 2024-11-16T04:26:49,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742428_1604 (size=12151) 2024-11-16T04:26:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:50,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731270083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731270083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731270084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,085 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731270293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731270293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,390 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a0a48b28bc9849b78058ba2e23c5acfc 2024-11-16T04:26:50,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/1c78f646f89e4c70b62f099ac023fc6e is 50, key is test_row_0/C:col10/1731731207890/Put/seqid=0 2024-11-16T04:26:50,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742429_1605 (size=12151) 2024-11-16T04:26:50,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731270604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731270604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:50,605 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:50,798 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/1c78f646f89e4c70b62f099ac023fc6e 2024-11-16T04:26:50,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/995cc21c45e541058c087ad89860a868 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868 2024-11-16T04:26:50,803 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868, entries=150, sequenceid=244, filesize=30.4 K 2024-11-16T04:26:50,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/a0a48b28bc9849b78058ba2e23c5acfc as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a0a48b28bc9849b78058ba2e23c5acfc 2024-11-16T04:26:50,806 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a0a48b28bc9849b78058ba2e23c5acfc, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T04:26:50,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/1c78f646f89e4c70b62f099ac023fc6e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/1c78f646f89e4c70b62f099ac023fc6e 2024-11-16T04:26:50,810 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/1c78f646f89e4c70b62f099ac023fc6e, entries=150, sequenceid=244, filesize=11.9 K 2024-11-16T04:26:50,810 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c16a25941e3d02bc3deccfcca062c9f6 in 1256ms, sequenceid=244, compaction requested=false 2024-11-16T04:26:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T04:26:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T04:26:50,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T04:26:50,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9550 sec 2024-11-16T04:26:50,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.9590 sec 2024-11-16T04:26:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:51,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:51,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-16T04:26:51,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8794dda2e534dbca029e0edd40c8c1d_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:51,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742430_1606 (size=12404) 2024-11-16T04:26:51,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731271157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731271159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,163 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731271264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731271273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731271473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731271484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,524 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:51,526 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c8794dda2e534dbca029e0edd40c8c1d_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8794dda2e534dbca029e0edd40c8c1d_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:51,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/eeb885dd6439440f8c62e4c8552d58e4, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:51,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/eeb885dd6439440f8c62e4c8552d58e4 is 175, key is test_row_0/A:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:51,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742431_1607 (size=31205) 2024-11-16T04:26:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731271784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731271793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:51,931 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/eeb885dd6439440f8c62e4c8552d58e4 2024-11-16T04:26:51,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/669155da005443a596b36a096911399f is 50, key is test_row_0/B:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:51,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742432_1608 (size=12251) 2024-11-16T04:26:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T04:26:52,024 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:52,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T04:26:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T04:26:52,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T04:26:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T04:26:52,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T04:26:52,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T04:26:52,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:52,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48134 deadline: 1731731272103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:52,106 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fda3efb342d8,45455,1731731034087 for 'test_row_0' in c16a25941e3d02bc3deccfcca062c9f6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T04:26:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:52,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T04:26:52,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T04:26:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731272293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:52,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731272303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:52,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T04:26:52,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,337 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T04:26:52,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/669155da005443a596b36a096911399f 2024-11-16T04:26:52,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/4c0a4890c482463e8b773a3f1b5b13a7 is 50, key is test_row_0/C:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T04:26:52,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742433_1609 (size=12251) 2024-11-16T04:26:52,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T04:26:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T04:26:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T04:26:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T04:26:52,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/4c0a4890c482463e8b773a3f1b5b13a7 2024-11-16T04:26:52,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/eeb885dd6439440f8c62e4c8552d58e4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4 2024-11-16T04:26:52,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4, entries=150, sequenceid=259, filesize=30.5 K 2024-11-16T04:26:52,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/669155da005443a596b36a096911399f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/669155da005443a596b36a096911399f 2024-11-16T04:26:52,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/669155da005443a596b36a096911399f, entries=150, sequenceid=259, filesize=12.0 K 2024-11-16T04:26:52,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/4c0a4890c482463e8b773a3f1b5b13a7 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/4c0a4890c482463e8b773a3f1b5b13a7 2024-11-16T04:26:52,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/4c0a4890c482463e8b773a3f1b5b13a7, entries=150, sequenceid=259, filesize=12.0 K 2024-11-16T04:26:52,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c16a25941e3d02bc3deccfcca062c9f6 in 1643ms, sequenceid=259, compaction requested=true 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:52,759 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:52,759 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:52,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:52,759 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:52,759 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:52,759 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/B is initiating minor compaction (all files) 2024-11-16T04:26:52,759 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/A is initiating minor compaction (all files) 2024-11-16T04:26:52,759 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/A in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,759 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/B in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,760 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c823aaf944b2418aac8a880ff9e54938, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=91.7 K 2024-11-16T04:26:52,760 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/15a06e6abee0412f8c78730945e6f368, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a0a48b28bc9849b78058ba2e23c5acfc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/669155da005443a596b36a096911399f] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=36.2 K 2024-11-16T04:26:52,760 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c823aaf944b2418aac8a880ff9e54938, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4] 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a06e6abee0412f8c78730945e6f368, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731731207789 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting c823aaf944b2418aac8a880ff9e54938, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731731207789 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a48b28bc9849b78058ba2e23c5acfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731731207890 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 995cc21c45e541058c087ad89860a868, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731731207890 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 669155da005443a596b36a096911399f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731731209964 2024-11-16T04:26:52,760 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb885dd6439440f8c62e4c8552d58e4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731731209964 2024-11-16T04:26:52,766 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:52,766 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/e3a87645ad2f45b2998c15a6eb4ac0fd is 50, key is test_row_0/B:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:52,767 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:52,769 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116feac0da0c2d24ac6be4d17331598774c_c16a25941e3d02bc3deccfcca062c9f6 store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:52,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742434_1610 (size=12865) 2024-11-16T04:26:52,770 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116feac0da0c2d24ac6be4d17331598774c_c16a25941e3d02bc3deccfcca062c9f6, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:52,771 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116feac0da0c2d24ac6be4d17331598774c_c16a25941e3d02bc3deccfcca062c9f6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:52,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742435_1611 (size=4469) 2024-11-16T04:26:52,775 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#A#compaction#517 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:52,775 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/ffe4544614e340b9b60880b52a785d61 is 175, key is test_row_0/A:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:52,775 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/e3a87645ad2f45b2998c15a6eb4ac0fd as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e3a87645ad2f45b2998c15a6eb4ac0fd 2024-11-16T04:26:52,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742436_1612 (size=31819) 2024-11-16T04:26:52,781 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/B of c16a25941e3d02bc3deccfcca062c9f6 into e3a87645ad2f45b2998c15a6eb4ac0fd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:52,781 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:52,781 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/B, priority=13, startTime=1731731212759; duration=0sec 2024-11-16T04:26:52,781 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:52,781 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:B 2024-11-16T04:26:52,781 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:52,782 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:52,782 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/C is initiating minor compaction (all files) 2024-11-16T04:26:52,782 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/C in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,782 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/ffe4544614e340b9b60880b52a785d61 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/ffe4544614e340b9b60880b52a785d61 2024-11-16T04:26:52,782 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/adf7359bd81540acbaef68ac710f7935, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/1c78f646f89e4c70b62f099ac023fc6e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/4c0a4890c482463e8b773a3f1b5b13a7] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=36.2 K 2024-11-16T04:26:52,782 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting adf7359bd81540acbaef68ac710f7935, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731731207789 2024-11-16T04:26:52,783 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c78f646f89e4c70b62f099ac023fc6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731731207890 2024-11-16T04:26:52,783 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c0a4890c482463e8b773a3f1b5b13a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731731209964 2024-11-16T04:26:52,785 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/A of c16a25941e3d02bc3deccfcca062c9f6 into ffe4544614e340b9b60880b52a785d61(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:52,785 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:52,785 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/A, priority=13, startTime=1731731212759; duration=0sec 2024-11-16T04:26:52,785 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:52,785 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:A 2024-11-16T04:26:52,787 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#C#compaction#518 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:52,787 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/cc27098b90eb47a8ab03f3a70ceddc43 is 50, key is test_row_0/C:col10/1731731209964/Put/seqid=0 2024-11-16T04:26:52,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45455 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T04:26:52,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:52,788 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-16T04:26:52,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742437_1613 (size=12865) 2024-11-16T04:26:52,795 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/cc27098b90eb47a8ab03f3a70ceddc43 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cc27098b90eb47a8ab03f3a70ceddc43 2024-11-16T04:26:52,799 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/C of c16a25941e3d02bc3deccfcca062c9f6 into cc27098b90eb47a8ab03f3a70ceddc43(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:52,799 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:52,799 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/C, priority=13, startTime=1731731212759; duration=0sec 2024-11-16T04:26:52,799 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:52,799 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:C 2024-11-16T04:26:52,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b2a208694b6440f99e421d3b17421da0_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731211158/Put/seqid=0 2024-11-16T04:26:52,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742438_1614 (size=12454) 2024-11-16T04:26:52,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:52,831 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116b2a208694b6440f99e421d3b17421da0_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b2a208694b6440f99e421d3b17421da0_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:52,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8823bef876e44685996c974f19d07151, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:52,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8823bef876e44685996c974f19d07151 is 175, key is test_row_0/A:col10/1731731211158/Put/seqid=0 2024-11-16T04:26:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742439_1615 (size=31255) 2024-11-16T04:26:53,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T04:26:53,237 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8823bef876e44685996c974f19d07151 2024-11-16T04:26:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/24bb3febcacd4998a96cf97104b59440 is 50, key is test_row_0/B:col10/1731731211158/Put/seqid=0 2024-11-16T04:26:53,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742440_1616 (size=12301) 2024-11-16T04:26:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:53,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. as already flushing 2024-11-16T04:26:53,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:53,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731273332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:53,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731273334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731273444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:53,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731273444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:53,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/24bb3febcacd4998a96cf97104b59440 2024-11-16T04:26:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/f21505e5aae34c83ac44272c177073f6 is 50, key is test_row_0/C:col10/1731731211158/Put/seqid=0 2024-11-16T04:26:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48138 deadline: 1731731273653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T04:26:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45455 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48144 deadline: 1731731273653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 , the old value is region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T04:26:53,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., hostname=fda3efb342d8,45455,1731731034087, seqNum=5 because the exception is null or not the one we care about 2024-11-16T04:26:53,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742441_1617 (size=12301) 2024-11-16T04:26:53,656 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/f21505e5aae34c83ac44272c177073f6 2024-11-16T04:26:53,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/8823bef876e44685996c974f19d07151 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151 2024-11-16T04:26:53,664 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151, entries=150, sequenceid=284, filesize=30.5 K 2024-11-16T04:26:53,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/24bb3febcacd4998a96cf97104b59440 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/24bb3febcacd4998a96cf97104b59440 2024-11-16T04:26:53,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/24bb3febcacd4998a96cf97104b59440, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T04:26:53,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/f21505e5aae34c83ac44272c177073f6 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f21505e5aae34c83ac44272c177073f6 2024-11-16T04:26:53,669 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f21505e5aae34c83ac44272c177073f6, entries=150, sequenceid=284, filesize=12.0 K 2024-11-16T04:26:53,670 INFO [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c16a25941e3d02bc3deccfcca062c9f6 in 882ms, sequenceid=284, compaction requested=false 2024-11-16T04:26:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:53,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fda3efb342d8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T04:26:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T04:26:53,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T04:26:53,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-11-16T04:26:53,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.6490 sec 2024-11-16T04:26:53,794 INFO [Thread-2406 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2406. 2024-11-16T04:26:53,795 DEBUG [Thread-2406 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,795 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,795 DEBUG [Thread-2406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,795 INFO [Thread-2404 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2404. 2024-11-16T04:26:53,795 DEBUG [Thread-2404 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,795 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,795 DEBUG [Thread-2404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,796 INFO [Thread-2412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2412. 2024-11-16T04:26:53,796 DEBUG [Thread-2412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,796 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,796 DEBUG [Thread-2412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,797 INFO [Thread-2410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2410. 2024-11-16T04:26:53,797 DEBUG [Thread-2410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,797 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,797 DEBUG [Thread-2410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,797 INFO [Thread-2408 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2408. 2024-11-16T04:26:53,797 DEBUG [Thread-2408 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,797 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,797 DEBUG [Thread-2408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45455 {}] regionserver.HRegion(8855): Flush requested on c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:53,968 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-16T04:26:53,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-16T04:26:53,968 INFO [Thread-2397 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2397. 2024-11-16T04:26:53,968 INFO [Thread-2399 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2399. 2024-11-16T04:26:53,968 DEBUG [Thread-2397 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,968 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,968 DEBUG [Thread-2399 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:53,968 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,968 DEBUG [Thread-2397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,969 DEBUG [Thread-2399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:53,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:53,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111647aac1197f844c36a354ca8bd919c1c0_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:53,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742442_1618 (size=12454) 2024-11-16T04:26:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T04:26:54,175 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:54,380 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:54,385 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111647aac1197f844c36a354ca8bd919c1c0_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111647aac1197f844c36a354ca8bd919c1c0_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:54,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d7bce94352f4472852b0dd72abba32c, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:54,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d7bce94352f4472852b0dd72abba32c is 175, key is test_row_0/A:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:54,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742443_1619 (size=31255) 2024-11-16T04:26:54,394 INFO [Thread-2395 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2395. 2024-11-16T04:26:54,394 DEBUG [Thread-2395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:54,394 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:54,394 DEBUG [Thread-2395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:54,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:54,475 INFO [Thread-2401 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2401. 2024-11-16T04:26:54,475 DEBUG [Thread-2401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:54,475 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:54,475 DEBUG [Thread-2401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:54,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:54,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d7bce94352f4472852b0dd72abba32c 2024-11-16T04:26:54,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/246017902247490197d816b569569e0e is 50, key is test_row_0/B:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:54,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742444_1620 (size=12301) 2024-11-16T04:26:55,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/246017902247490197d816b569569e0e 2024-11-16T04:26:55,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/29886249b1ee48e7b05520645e4aa483 is 50, key is test_row_0/C:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:55,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742445_1621 (size=12301) 2024-11-16T04:26:55,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/29886249b1ee48e7b05520645e4aa483 2024-11-16T04:26:55,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/2d7bce94352f4472852b0dd72abba32c as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c 2024-11-16T04:26:55,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c, entries=150, sequenceid=300, filesize=30.5 K 2024-11-16T04:26:55,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/246017902247490197d816b569569e0e as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/246017902247490197d816b569569e0e 2024-11-16T04:26:55,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/246017902247490197d816b569569e0e, entries=150, sequenceid=300, filesize=12.0 K 2024-11-16T04:26:55,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/29886249b1ee48e7b05520645e4aa483 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/29886249b1ee48e7b05520645e4aa483 2024-11-16T04:26:55,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/29886249b1ee48e7b05520645e4aa483, entries=150, sequenceid=300, filesize=12.0 K 2024-11-16T04:26:55,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=13.42 KB/13740 for c16a25941e3d02bc3deccfcca062c9f6 in 1681ms, sequenceid=300, compaction requested=true 2024-11-16T04:26:55,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T04:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T04:26:55,650 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c16a25941e3d02bc3deccfcca062c9f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T04:26:55,650 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/B is initiating minor compaction (all files) 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/A is initiating minor compaction (all files) 2024-11-16T04:26:55,651 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/B in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:55,651 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/A in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:55,651 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e3a87645ad2f45b2998c15a6eb4ac0fd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/24bb3febcacd4998a96cf97104b59440, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/246017902247490197d816b569569e0e] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=36.6 K 2024-11-16T04:26:55,651 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/ffe4544614e340b9b60880b52a785d61, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=92.1 K 2024-11-16T04:26:55,651 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. files: [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/ffe4544614e340b9b60880b52a785d61, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c] 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a87645ad2f45b2998c15a6eb4ac0fd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731731209964 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffe4544614e340b9b60880b52a785d61, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731731209964 2024-11-16T04:26:55,651 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 24bb3febcacd4998a96cf97104b59440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731731211152 2024-11-16T04:26:55,652 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8823bef876e44685996c974f19d07151, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731731211152 2024-11-16T04:26:55,652 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 246017902247490197d816b569569e0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731731213331 2024-11-16T04:26:55,652 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7bce94352f4472852b0dd72abba32c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731731213331 2024-11-16T04:26:55,657 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:55,658 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:55,658 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/ad9b97cdf9b545389116558211e9b50b is 50, key is test_row_0/B:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:55,658 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d1938792a20842aa826ba7ac54cc4dbd_c16a25941e3d02bc3deccfcca062c9f6 store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:55,661 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d1938792a20842aa826ba7ac54cc4dbd_c16a25941e3d02bc3deccfcca062c9f6, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:55,661 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d1938792a20842aa826ba7ac54cc4dbd_c16a25941e3d02bc3deccfcca062c9f6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:55,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742446_1622 (size=13017) 2024-11-16T04:26:55,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742447_1623 (size=4469) 2024-11-16T04:26:55,665 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#A#compaction#526 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:55,665 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d1c341eef2a243e6bea964638ae135c2 is 175, key is test_row_0/A:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:55,667 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/ad9b97cdf9b545389116558211e9b50b as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/ad9b97cdf9b545389116558211e9b50b 2024-11-16T04:26:55,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742448_1624 (size=31971) 2024-11-16T04:26:55,671 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/B of c16a25941e3d02bc3deccfcca062c9f6 into ad9b97cdf9b545389116558211e9b50b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:55,671 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:55,671 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/B, priority=13, startTime=1731731215650; duration=0sec 2024-11-16T04:26:55,671 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T04:26:55,671 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:B 2024-11-16T04:26:55,671 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T04:26:55,671 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T04:26:55,671 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1541): c16a25941e3d02bc3deccfcca062c9f6/C is initiating minor compaction (all files) 2024-11-16T04:26:55,672 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c16a25941e3d02bc3deccfcca062c9f6/C in TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:55,672 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cc27098b90eb47a8ab03f3a70ceddc43, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f21505e5aae34c83ac44272c177073f6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/29886249b1ee48e7b05520645e4aa483] into tmpdir=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp, totalSize=36.6 K 2024-11-16T04:26:55,672 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting cc27098b90eb47a8ab03f3a70ceddc43, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731731209964 2024-11-16T04:26:55,672 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting f21505e5aae34c83ac44272c177073f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731731211152 2024-11-16T04:26:55,672 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] compactions.Compactor(225): Compacting 29886249b1ee48e7b05520645e4aa483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731731213331 2024-11-16T04:26:55,677 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c16a25941e3d02bc3deccfcca062c9f6#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T04:26:55,677 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/dd118574194b44e8a711c530a23f8d34 is 50, key is test_row_0/C:col10/1731731213331/Put/seqid=0 2024-11-16T04:26:55,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742449_1625 (size=13017) 2024-11-16T04:26:56,079 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/d1c341eef2a243e6bea964638ae135c2 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d1c341eef2a243e6bea964638ae135c2 2024-11-16T04:26:56,084 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/A of c16a25941e3d02bc3deccfcca062c9f6 into d1c341eef2a243e6bea964638ae135c2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:56,084 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:56,084 INFO [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/A, priority=13, startTime=1731731215650; duration=0sec 2024-11-16T04:26:56,085 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:56,085 DEBUG [RS:0;fda3efb342d8:45455-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:A 2024-11-16T04:26:56,085 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/dd118574194b44e8a711c530a23f8d34 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/dd118574194b44e8a711c530a23f8d34 2024-11-16T04:26:56,090 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c16a25941e3d02bc3deccfcca062c9f6/C of c16a25941e3d02bc3deccfcca062c9f6 into dd118574194b44e8a711c530a23f8d34(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T04:26:56,090 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c16a25941e3d02bc3deccfcca062c9f6: 2024-11-16T04:26:56,090 INFO [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., storeName=c16a25941e3d02bc3deccfcca062c9f6/C, priority=13, startTime=1731731215650; duration=0sec 2024-11-16T04:26:56,090 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T04:26:56,090 DEBUG [RS:0;fda3efb342d8:45455-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c16a25941e3d02bc3deccfcca062c9f6:C 2024-11-16T04:26:56,157 INFO [Thread-2393 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2393. 2024-11-16T04:26:56,157 DEBUG [Thread-2393 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T04:26:56,158 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:56,158 DEBUG [Thread-2393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:56,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:56,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T04:26:56,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T04:26:56,158 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3874 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3909 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3723 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3878 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3914 2024-11-16T04:26:56,159 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T04:26:56,159 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:26:56,159 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:26:56,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:56,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:56,160 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:56,161 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T04:26:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:56,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T04:26:56,166 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731216165"}]},"ts":"1731731216165"} 2024-11-16T04:26:56,167 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T04:26:56,168 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T04:26:56,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T04:26:56,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, UNASSIGN}] 2024-11-16T04:26:56,171 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, UNASSIGN 2024-11-16T04:26:56,172 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=CLOSING, regionLocation=fda3efb342d8,45455,1731731034087 2024-11-16T04:26:56,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, UNASSIGN because future has completed 2024-11-16T04:26:56,174 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T04:26:56,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087}] 2024-11-16T04:26:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T04:26:56,328 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:56,328 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T04:26:56,328 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing c16a25941e3d02bc3deccfcca062c9f6, disabling compactions & flushes 2024-11-16T04:26:56,328 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:56,328 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:56,328 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. after waiting 0 ms 2024-11-16T04:26:56,328 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:56,329 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing c16a25941e3d02bc3deccfcca062c9f6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-16T04:26:56,340 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111622643a87229445629f200190fbf19e07_c16a25941e3d02bc3deccfcca062c9f6 is 50, key is test_row_0/A:col10/1731731216155/Put/seqid=0 2024-11-16T04:26:56,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742450_1626 (size=9914) 2024-11-16T04:26:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T04:26:56,744 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T04:26:56,753 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111622643a87229445629f200190fbf19e07_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111622643a87229445629f200190fbf19e07_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:56,754 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/6a4d64c717f04cc6b2d0f02b94fc3a9f, store: [table=TestAcidGuarantees family=A region=c16a25941e3d02bc3deccfcca062c9f6] 2024-11-16T04:26:56,754 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/6a4d64c717f04cc6b2d0f02b94fc3a9f is 175, key is test_row_0/A:col10/1731731216155/Put/seqid=0 2024-11-16T04:26:56,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742451_1627 (size=22561) 2024-11-16T04:26:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T04:26:57,159 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/6a4d64c717f04cc6b2d0f02b94fc3a9f 2024-11-16T04:26:57,174 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/fdc620486ca542888a60d9e0963857e0 is 50, key is test_row_0/B:col10/1731731216155/Put/seqid=0 2024-11-16T04:26:57,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742452_1628 (size=9857) 2024-11-16T04:26:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T04:26:57,578 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/fdc620486ca542888a60d9e0963857e0 2024-11-16T04:26:57,590 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/3dba505390fb4116a59c6f4c8e3ccd0a is 50, key is test_row_0/C:col10/1731731216155/Put/seqid=0 2024-11-16T04:26:57,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742453_1629 (size=9857) 2024-11-16T04:26:57,996 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/3dba505390fb4116a59c6f4c8e3ccd0a 2024-11-16T04:26:58,005 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/A/6a4d64c717f04cc6b2d0f02b94fc3a9f as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/6a4d64c717f04cc6b2d0f02b94fc3a9f 2024-11-16T04:26:58,009 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/6a4d64c717f04cc6b2d0f02b94fc3a9f, entries=100, sequenceid=309, filesize=22.0 K 2024-11-16T04:26:58,010 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/B/fdc620486ca542888a60d9e0963857e0 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/fdc620486ca542888a60d9e0963857e0 2024-11-16T04:26:58,014 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/fdc620486ca542888a60d9e0963857e0, entries=100, sequenceid=309, filesize=9.6 K 2024-11-16T04:26:58,015 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/.tmp/C/3dba505390fb4116a59c6f4c8e3ccd0a as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/3dba505390fb4116a59c6f4c8e3ccd0a 2024-11-16T04:26:58,018 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/3dba505390fb4116a59c6f4c8e3ccd0a, entries=100, sequenceid=309, filesize=9.6 K 2024-11-16T04:26:58,019 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c16a25941e3d02bc3deccfcca062c9f6 in 1691ms, sequenceid=309, compaction requested=false 2024-11-16T04:26:58,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/82bece34cdfe4f2f862d97bc28b6fe6b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9cd278312b1d467bb23241036e492142, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/09854141b8c0472a8233439af3753611, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c823aaf944b2418aac8a880ff9e54938, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/ffe4544614e340b9b60880b52a785d61, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c] to archive 2024-11-16T04:26:58,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:26:58,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d1397886ab84cceb60cdea2ce480707 2024-11-16T04:26:58,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8052fc76f019401690df28905e672471 2024-11-16T04:26:58,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/f80a6c01d13a4be4bfad63fb0dba79ef 2024-11-16T04:26:58,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c37236e39d124762b1f35bc066c1c21e 2024-11-16T04:26:58,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/36ac8e5dcd99411eae0ecc7f6c2a610a 2024-11-16T04:26:58,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9d95f4a042df40a9acf61df77710f20e 2024-11-16T04:26:58,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/82bece34cdfe4f2f862d97bc28b6fe6b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/82bece34cdfe4f2f862d97bc28b6fe6b 2024-11-16T04:26:58,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/efd05243140c4ddeada27df360ffcab1 2024-11-16T04:26:58,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/3dd940b988044e20b74a7a7b3ecfb268 2024-11-16T04:26:58,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9cd278312b1d467bb23241036e492142 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/9cd278312b1d467bb23241036e492142 2024-11-16T04:26:58,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/7796b8aebb6e4fc39271b115a3a2df27 2024-11-16T04:26:58,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/da81dbd5131c4da8ad5ce176c2012142 2024-11-16T04:26:58,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/09854141b8c0472a8233439af3753611 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/09854141b8c0472a8233439af3753611 2024-11-16T04:26:58,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d36deb3ef7034707a61a2d68b512b851 2024-11-16T04:26:58,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d0b97b7de4a4493aa3ba8d377b1dbe5e 2024-11-16T04:26:58,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c823aaf944b2418aac8a880ff9e54938 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/c823aaf944b2418aac8a880ff9e54938 2024-11-16T04:26:58,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/995cc21c45e541058c087ad89860a868 2024-11-16T04:26:58,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/ffe4544614e340b9b60880b52a785d61 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/ffe4544614e340b9b60880b52a785d61 2024-11-16T04:26:58,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/eeb885dd6439440f8c62e4c8552d58e4 2024-11-16T04:26:58,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/8823bef876e44685996c974f19d07151 2024-11-16T04:26:58,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/2d7bce94352f4472852b0dd72abba32c 2024-11-16T04:26:58,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/7c59c13cad8341c1bfef8ec6fe16e012, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a78d38d6e65c4609a54a2734d177c2be, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/217f1a4c9db14473be205ed67e5f96a0, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/0d7a9c866e8c49259ad787e889cb4300, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/9016ca7af9e74517bff0cd26c3149b7e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e7a4b1b3db3a4dc1b50944cbadcd149e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/721f405d8aa04a9eb747c9c1f5900f77, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/3693f45be6db488fb40b424a7e544680, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/d8814b8a4256434ea197d88aaf1d483a, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a32854f9f8794486ad22f79b6daf1e0e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/439e83c546114ca7a8b63f9a0421dd93, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/14f7bee99ef74608879aaf219bd9052b, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a1717ac72a0343b8a3b4b73902510e48, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/15a06e6abee0412f8c78730945e6f368, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/cb6755a7802549658170d3dd1b64f383, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a0a48b28bc9849b78058ba2e23c5acfc, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e3a87645ad2f45b2998c15a6eb4ac0fd, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/669155da005443a596b36a096911399f, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/24bb3febcacd4998a96cf97104b59440, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/246017902247490197d816b569569e0e] to archive 2024-11-16T04:26:58,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:26:58,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/7c59c13cad8341c1bfef8ec6fe16e012 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/7c59c13cad8341c1bfef8ec6fe16e012 2024-11-16T04:26:58,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a78d38d6e65c4609a54a2734d177c2be to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a78d38d6e65c4609a54a2734d177c2be 2024-11-16T04:26:58,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/217f1a4c9db14473be205ed67e5f96a0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/217f1a4c9db14473be205ed67e5f96a0 2024-11-16T04:26:58,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/0d7a9c866e8c49259ad787e889cb4300 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/0d7a9c866e8c49259ad787e889cb4300 2024-11-16T04:26:58,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/9016ca7af9e74517bff0cd26c3149b7e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/9016ca7af9e74517bff0cd26c3149b7e 2024-11-16T04:26:58,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e7a4b1b3db3a4dc1b50944cbadcd149e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e7a4b1b3db3a4dc1b50944cbadcd149e 2024-11-16T04:26:58,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/721f405d8aa04a9eb747c9c1f5900f77 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/721f405d8aa04a9eb747c9c1f5900f77 2024-11-16T04:26:58,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/eaa343ebd57b4f2dbe33f6f86ad3f1c4 2024-11-16T04:26:58,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/3693f45be6db488fb40b424a7e544680 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/3693f45be6db488fb40b424a7e544680 2024-11-16T04:26:58,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/d8814b8a4256434ea197d88aaf1d483a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/d8814b8a4256434ea197d88aaf1d483a 2024-11-16T04:26:58,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a32854f9f8794486ad22f79b6daf1e0e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a32854f9f8794486ad22f79b6daf1e0e 2024-11-16T04:26:58,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/439e83c546114ca7a8b63f9a0421dd93 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/439e83c546114ca7a8b63f9a0421dd93 2024-11-16T04:26:58,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/14f7bee99ef74608879aaf219bd9052b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/14f7bee99ef74608879aaf219bd9052b 2024-11-16T04:26:58,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a1717ac72a0343b8a3b4b73902510e48 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a1717ac72a0343b8a3b4b73902510e48 2024-11-16T04:26:58,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/15a06e6abee0412f8c78730945e6f368 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/15a06e6abee0412f8c78730945e6f368 2024-11-16T04:26:58,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/cb6755a7802549658170d3dd1b64f383 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/cb6755a7802549658170d3dd1b64f383 2024-11-16T04:26:58,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a0a48b28bc9849b78058ba2e23c5acfc to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/a0a48b28bc9849b78058ba2e23c5acfc 2024-11-16T04:26:58,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e3a87645ad2f45b2998c15a6eb4ac0fd to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/e3a87645ad2f45b2998c15a6eb4ac0fd 2024-11-16T04:26:58,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/669155da005443a596b36a096911399f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/669155da005443a596b36a096911399f 2024-11-16T04:26:58,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/24bb3febcacd4998a96cf97104b59440 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/24bb3febcacd4998a96cf97104b59440 2024-11-16T04:26:58,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/246017902247490197d816b569569e0e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/246017902247490197d816b569569e0e 2024-11-16T04:26:58,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f8e84ed478b44adc9c12d57e03ad8ebb, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/eb347b2634e64612ac970d74042d9a59, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/fa01a09775934c8a883055f7e8489e8d, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/73000bfbbf0646389b6c5ec2878c5f57, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/857bb4c3e8a74853bace93ed2c68f215, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/610b6ff06d66441eb166cfe68a0180b7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/18b19e7302424fe09b8b8d640691ba25, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2a205aa7218841e99efb779c2359cc83, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/13837645178143a1ae88505941807ad5, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/b69c9e98deab4929a10681f4e80bab48, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cb442f0ec9fa48cb9b8f710bd43049ee, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/682e7f784c4b4b419ae0847a36cd4c27, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/bcc03dd3f67444eb9a374c045c60a2c4, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/44d3948b600e4c2c95516fbe7be63c7c, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/adf7359bd81540acbaef68ac710f7935, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2c3de6a81a7d4004b21a842e3c090dce, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/1c78f646f89e4c70b62f099ac023fc6e, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cc27098b90eb47a8ab03f3a70ceddc43, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/4c0a4890c482463e8b773a3f1b5b13a7, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f21505e5aae34c83ac44272c177073f6, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/29886249b1ee48e7b05520645e4aa483] to archive 2024-11-16T04:26:58,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T04:26:58,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f8e84ed478b44adc9c12d57e03ad8ebb to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f8e84ed478b44adc9c12d57e03ad8ebb 2024-11-16T04:26:58,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/eb347b2634e64612ac970d74042d9a59 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/eb347b2634e64612ac970d74042d9a59 2024-11-16T04:26:58,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/fa01a09775934c8a883055f7e8489e8d to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/fa01a09775934c8a883055f7e8489e8d 2024-11-16T04:26:58,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/73000bfbbf0646389b6c5ec2878c5f57 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/73000bfbbf0646389b6c5ec2878c5f57 2024-11-16T04:26:58,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/857bb4c3e8a74853bace93ed2c68f215 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/857bb4c3e8a74853bace93ed2c68f215 2024-11-16T04:26:58,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/610b6ff06d66441eb166cfe68a0180b7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/610b6ff06d66441eb166cfe68a0180b7 2024-11-16T04:26:58,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/18b19e7302424fe09b8b8d640691ba25 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/18b19e7302424fe09b8b8d640691ba25 2024-11-16T04:26:58,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2a205aa7218841e99efb779c2359cc83 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2a205aa7218841e99efb779c2359cc83 2024-11-16T04:26:58,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/13837645178143a1ae88505941807ad5 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/13837645178143a1ae88505941807ad5 2024-11-16T04:26:58,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/b69c9e98deab4929a10681f4e80bab48 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/b69c9e98deab4929a10681f4e80bab48 2024-11-16T04:26:58,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cb442f0ec9fa48cb9b8f710bd43049ee to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cb442f0ec9fa48cb9b8f710bd43049ee 2024-11-16T04:26:58,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/682e7f784c4b4b419ae0847a36cd4c27 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/682e7f784c4b4b419ae0847a36cd4c27 2024-11-16T04:26:58,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/bcc03dd3f67444eb9a374c045c60a2c4 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/bcc03dd3f67444eb9a374c045c60a2c4 2024-11-16T04:26:58,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/44d3948b600e4c2c95516fbe7be63c7c to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/44d3948b600e4c2c95516fbe7be63c7c 2024-11-16T04:26:58,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/adf7359bd81540acbaef68ac710f7935 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/adf7359bd81540acbaef68ac710f7935 2024-11-16T04:26:58,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2c3de6a81a7d4004b21a842e3c090dce to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/2c3de6a81a7d4004b21a842e3c090dce 2024-11-16T04:26:58,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/1c78f646f89e4c70b62f099ac023fc6e to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/1c78f646f89e4c70b62f099ac023fc6e 2024-11-16T04:26:58,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cc27098b90eb47a8ab03f3a70ceddc43 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/cc27098b90eb47a8ab03f3a70ceddc43 2024-11-16T04:26:58,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/4c0a4890c482463e8b773a3f1b5b13a7 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/4c0a4890c482463e8b773a3f1b5b13a7 2024-11-16T04:26:58,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f21505e5aae34c83ac44272c177073f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/f21505e5aae34c83ac44272c177073f6 2024-11-16T04:26:58,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/29886249b1ee48e7b05520645e4aa483 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/29886249b1ee48e7b05520645e4aa483 2024-11-16T04:26:58,077 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/recovered.edits/312.seqid, newMaxSeqId=312, maxSeqId=4 2024-11-16T04:26:58,077 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. 2024-11-16T04:26:58,077 DEBUG [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for c16a25941e3d02bc3deccfcca062c9f6: Waiting for close lock at 1731731216328Running coprocessor pre-close hooks at 1731731216328Disabling compacts and flushes for region at 1731731216328Disabling writes for close at 1731731216328Obtaining lock to block concurrent updates at 1731731216329 (+1 ms)Preparing flush snapshotting stores in c16a25941e3d02bc3deccfcca062c9f6 at 1731731216329Finished memstore snapshotting TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731731216329Flushing stores of TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6. at 1731731216331 (+2 ms)Flushing c16a25941e3d02bc3deccfcca062c9f6/A: creating writer at 1731731216331Flushing mob file c16a25941e3d02bc3deccfcca062c9f6/A: appending metadata at 1731731216340 (+9 ms)Flushing mob file c16a25941e3d02bc3deccfcca062c9f6/A: closing flushed file at 1731731216340Flushing c16a25941e3d02bc3deccfcca062c9f6/A: appending metadata at 1731731216754 (+414 ms)Flushing c16a25941e3d02bc3deccfcca062c9f6/A: closing flushed file at 1731731216754Flushing c16a25941e3d02bc3deccfcca062c9f6/B: creating writer at 1731731217168 (+414 ms)Flushing c16a25941e3d02bc3deccfcca062c9f6/B: appending metadata at 1731731217173 (+5 ms)Flushing c16a25941e3d02bc3deccfcca062c9f6/B: closing flushed file at 1731731217173Flushing c16a25941e3d02bc3deccfcca062c9f6/C: creating writer at 1731731217587 (+414 ms)Flushing c16a25941e3d02bc3deccfcca062c9f6/C: appending metadata at 1731731217590 (+3 ms)Flushing c16a25941e3d02bc3deccfcca062c9f6/C: closing flushed file at 1731731217590Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62db6d25: reopening flushed file at 1731731218004 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23115407: reopening flushed file at 1731731218009 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@763fed3f: reopening flushed file at 1731731218014 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for c16a25941e3d02bc3deccfcca062c9f6 in 1691ms, sequenceid=309, compaction requested=false at 1731731218019 (+5 ms)Writing region close event to WAL at 1731731218075 (+56 ms)Running coprocessor post-close hooks at 1731731218077 (+2 ms)Closed at 1731731218077 2024-11-16T04:26:58,078 INFO [RS_CLOSE_REGION-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,079 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=c16a25941e3d02bc3deccfcca062c9f6, regionState=CLOSED 2024-11-16T04:26:58,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 because future has completed 2024-11-16T04:26:58,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T04:26:58,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure c16a25941e3d02bc3deccfcca062c9f6, server=fda3efb342d8,45455,1731731034087 in 1.9070 sec 2024-11-16T04:26:58,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-16T04:26:58,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c16a25941e3d02bc3deccfcca062c9f6, UNASSIGN in 1.9120 sec 2024-11-16T04:26:58,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T04:26:58,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9160 sec 2024-11-16T04:26:58,087 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731731218087"}]},"ts":"1731731218087"} 2024-11-16T04:26:58,088 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T04:26:58,088 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T04:26:58,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9280 sec 2024-11-16T04:26:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T04:26:58,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:58,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T04:26:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:58,320 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T04:26:58,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:58,324 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,327 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C, FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/recovered.edits] 2024-11-16T04:26:58,331 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/6a4d64c717f04cc6b2d0f02b94fc3a9f to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/6a4d64c717f04cc6b2d0f02b94fc3a9f 2024-11-16T04:26:58,332 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d1c341eef2a243e6bea964638ae135c2 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/A/d1c341eef2a243e6bea964638ae135c2 2024-11-16T04:26:58,336 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/ad9b97cdf9b545389116558211e9b50b to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/ad9b97cdf9b545389116558211e9b50b 2024-11-16T04:26:58,338 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/fdc620486ca542888a60d9e0963857e0 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/B/fdc620486ca542888a60d9e0963857e0 2024-11-16T04:26:58,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/3dba505390fb4116a59c6f4c8e3ccd0a to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/3dba505390fb4116a59c6f4c8e3ccd0a 2024-11-16T04:26:58,345 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/dd118574194b44e8a711c530a23f8d34 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/C/dd118574194b44e8a711c530a23f8d34 2024-11-16T04:26:58,350 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/recovered.edits/312.seqid to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6/recovered.edits/312.seqid 2024-11-16T04:26:58,350 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/default/TestAcidGuarantees/c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,351 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T04:26:58,351 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T04:26:58,353 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T04:26:58,358 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602a3756c3fb34d409f890f86a5769f9a_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111602a3756c3fb34d409f890f86a5769f9a_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,361 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111622643a87229445629f200190fbf19e07_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111622643a87229445629f200190fbf19e07_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,363 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111624f0dc0c68ab4ebc873748f877f0bbce_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111624f0dc0c68ab4ebc873748f877f0bbce_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,366 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116296a6546d8904915aed207c465e11c2a_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116296a6546d8904915aed207c465e11c2a_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,369 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162bf08549104840139cf4c03c2da41c43_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162bf08549104840139cf4c03c2da41c43_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,372 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162cb4205b2d874a8684ec560ef4130088_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162cb4205b2d874a8684ec560ef4130088_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,374 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111647aac1197f844c36a354ca8bd919c1c0_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111647aac1197f844c36a354ca8bd919c1c0_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,377 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116756d7b6818954850b671d539d0be8ede_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116756d7b6818954850b671d539d0be8ede_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,380 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111681b29a1cd57d4dcea3f02dee48ecc505_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111681b29a1cd57d4dcea3f02dee48ecc505_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,383 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116959bdbee86ca4a3ba10250e51cf35773_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116959bdbee86ca4a3ba10250e51cf35773_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,385 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b2a208694b6440f99e421d3b17421da0_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116b2a208694b6440f99e421d3b17421da0_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,387 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c2f3cdab60df40aeb71e145649f38156_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c2f3cdab60df40aeb71e145649f38156_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,388 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c7276a352a0742a29f09a68f15d26ead_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c7276a352a0742a29f09a68f15d26ead_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,390 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8794dda2e534dbca029e0edd40c8c1d_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c8794dda2e534dbca029e0edd40c8c1d_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,391 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d7bf898ba5bc4fd393c98c09dfb06920_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d7bf898ba5bc4fd393c98c09dfb06920_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb9fe2819e2f41e4b0ab0900027659d5_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116eb9fe2819e2f41e4b0ab0900027659d5_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f2b19a2d815a47a4b16023d073a4bef6_c16a25941e3d02bc3deccfcca062c9f6 to hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f2b19a2d815a47a4b16023d073a4bef6_c16a25941e3d02bc3deccfcca062c9f6 2024-11-16T04:26:58,395 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T04:26:58,398 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:58,402 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T04:26:58,404 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T04:26:58,406 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:58,406 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T04:26:58,406 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731731218406"}]},"ts":"9223372036854775807"} 2024-11-16T04:26:58,408 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T04:26:58,408 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c16a25941e3d02bc3deccfcca062c9f6, NAME => 'TestAcidGuarantees,,1731731190599.c16a25941e3d02bc3deccfcca062c9f6.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T04:26:58,408 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T04:26:58,409 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731731218408"}]},"ts":"9223372036854775807"} 2024-11-16T04:26:58,411 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T04:26:58,412 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T04:26:58,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 95 msec 2024-11-16T04:26:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T04:26:58,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T04:26:58,424 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T04:26:58,436 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=397 (was 432), ProcessCount=11 (was 11), AvailableMemoryMB=7944 (was 7995) 2024-11-16T04:26:58,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T04:26:58,437 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:26:58,437 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:26:58,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:58,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:58,437 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T04:26:58,437 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T04:26:58,437 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-16T04:26:58,438 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fda3efb342d8,45087,1731731033310 2024-11-16T04:26:58,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T04:26:58,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T04:26:58,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:26:58,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:26:58,509 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T04:26:58,510 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T04:26:58,511 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:26:58,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:58,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T04:26:58,512 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda3efb342d8,45455,1731731034087' ***** 2024-11-16T04:26:58,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T04:26:58,512 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T04:26:58,513 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T04:26:58,514 INFO [RS:0;fda3efb342d8:45455 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T04:26:58,514 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T04:26:58,514 INFO [RS:0;fda3efb342d8:45455 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T04:26:58,514 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(959): stopping server fda3efb342d8,45455,1731731034087 2024-11-16T04:26:58,514 INFO [RS:0;fda3efb342d8:45455 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T04:26:58,514 INFO [RS:0;fda3efb342d8:45455 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fda3efb342d8:45455. 2024-11-16T04:26:58,514 DEBUG [RS:0;fda3efb342d8:45455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T04:26:58,514 DEBUG [RS:0;fda3efb342d8:45455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:58,515 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T04:26:58,515 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T04:26:58,515 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T04:26:58,515 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T04:26:58,516 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T04:26:58,516 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T04:26:58,516 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:58,516 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T04:26:58,516 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T04:26:58,516 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T04:26:58,516 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T04:26:58,516 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T04:26:58,517 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T04:26:58,538 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/info/f8d0c68f6b8a4d9b9a06812f94faa8a5 is 95, key is TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a./info:/1731731065472/DeleteFamily/seqid=0 2024-11-16T04:26:58,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742454_1630 (size=6514) 2024-11-16T04:26:58,549 INFO [regionserver/fda3efb342d8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T04:26:58,716 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:58,917 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:58,942 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/info/f8d0c68f6b8a4d9b9a06812f94faa8a5 2024-11-16T04:26:58,960 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/ns/eabb694257834a1f88ba51cd9509a6e1 is 93, key is TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a./ns:/1731731065428/DeleteFamily/seqid=0 2024-11-16T04:26:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742455_1631 (size=5906) 2024-11-16T04:26:59,117 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:59,131 INFO [regionserver/fda3efb342d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T04:26:59,131 INFO [regionserver/fda3efb342d8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T04:26:59,317 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:59,363 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/ns/eabb694257834a1f88ba51cd9509a6e1 2024-11-16T04:26:59,386 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/rep_barrier/b140bad6f81443118aef72ab406f5982 is 102, key is TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a./rep_barrier:/1731731065428/DeleteFamily/seqid=0 2024-11-16T04:26:59,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742456_1632 (size=6025) 2024-11-16T04:26:59,391 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/rep_barrier/b140bad6f81443118aef72ab406f5982 2024-11-16T04:26:59,419 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/table/ea6c884e3acd404a809211c92dd15a9d is 96, key is TestAcidGuarantees,,1731731038466.20bf25e458f1c8ea7c07f170b9f8e62a./table:/1731731065428/DeleteFamily/seqid=0 2024-11-16T04:26:59,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742457_1633 (size=5936) 2024-11-16T04:26:59,517 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:59,717 DEBUG [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T04:26:59,823 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/table/ea6c884e3acd404a809211c92dd15a9d 2024-11-16T04:26:59,826 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/info/f8d0c68f6b8a4d9b9a06812f94faa8a5 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/info/f8d0c68f6b8a4d9b9a06812f94faa8a5 2024-11-16T04:26:59,829 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/info/f8d0c68f6b8a4d9b9a06812f94faa8a5, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T04:26:59,830 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/ns/eabb694257834a1f88ba51cd9509a6e1 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/ns/eabb694257834a1f88ba51cd9509a6e1 2024-11-16T04:26:59,833 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/ns/eabb694257834a1f88ba51cd9509a6e1, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T04:26:59,833 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/rep_barrier/b140bad6f81443118aef72ab406f5982 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/rep_barrier/b140bad6f81443118aef72ab406f5982 2024-11-16T04:26:59,836 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/rep_barrier/b140bad6f81443118aef72ab406f5982, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T04:26:59,837 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/.tmp/table/ea6c884e3acd404a809211c92dd15a9d as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/table/ea6c884e3acd404a809211c92dd15a9d 2024-11-16T04:26:59,840 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/table/ea6c884e3acd404a809211c92dd15a9d, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T04:26:59,841 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1325ms, sequenceid=90, compaction requested=false 2024-11-16T04:26:59,845 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T04:26:59,846 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T04:26:59,846 INFO [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T04:26:59,846 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731731218516Running coprocessor pre-close hooks at 1731731218516Disabling compacts and flushes for region at 1731731218516Disabling writes for close at 1731731218516Obtaining lock to block concurrent updates at 1731731218517 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731731218517Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731731218517Flushing stores of hbase:meta,,1.1588230740 at 1731731218518 (+1 ms)Flushing 1588230740/info: creating writer at 1731731218518Flushing 1588230740/info: appending metadata at 1731731218538 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731731218538Flushing 1588230740/ns: creating writer at 1731731218946 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731731218960 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731731218960Flushing 1588230740/rep_barrier: creating writer at 1731731219366 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731731219386 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731731219386Flushing 1588230740/table: creating writer at 1731731219394 (+8 ms)Flushing 1588230740/table: appending metadata at 1731731219419 (+25 ms)Flushing 1588230740/table: closing flushed file at 1731731219419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@389c453a: reopening flushed file at 1731731219825 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62a6d6f2: reopening flushed file at 1731731219829 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bd06a47: reopening flushed file at 1731731219833 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e729d33: reopening flushed file at 1731731219836 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1325ms, sequenceid=90, compaction requested=false at 1731731219841 (+5 ms)Writing region close event to WAL at 1731731219843 (+2 ms)Running coprocessor post-close hooks at 1731731219846 (+3 ms)Closed at 1731731219846 2024-11-16T04:26:59,846 DEBUG [RS_CLOSE_META-regionserver/fda3efb342d8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T04:26:59,918 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(976): stopping server fda3efb342d8,45455,1731731034087; all regions closed. 2024-11-16T04:26:59,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741834_1010 (size=25344) 2024-11-16T04:26:59,925 DEBUG [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/oldWALs 2024-11-16T04:26:59,925 INFO [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda3efb342d8%2C45455%2C1731731034087.meta:.meta(num 1731731037917) 2024-11-16T04:26:59,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741833_1009 (size=14239713) 2024-11-16T04:26:59,928 DEBUG [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/oldWALs 2024-11-16T04:26:59,928 INFO [RS:0;fda3efb342d8:45455 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda3efb342d8%2C45455%2C1731731034087:(num 1731731037292) 2024-11-16T04:26:59,928 DEBUG [RS:0;fda3efb342d8:45455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T04:26:59,928 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T04:26:59,929 INFO [RS:0;fda3efb342d8:45455 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T04:26:59,929 INFO [RS:0;fda3efb342d8:45455 {}] hbase.ChoreService(370): Chore service for: regionserver/fda3efb342d8:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T04:26:59,929 INFO [RS:0;fda3efb342d8:45455 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T04:26:59,929 INFO [regionserver/fda3efb342d8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T04:26:59,929 INFO [RS:0;fda3efb342d8:45455 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45455 2024-11-16T04:27:00,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda3efb342d8,45455,1731731034087 2024-11-16T04:27:00,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T04:27:00,093 INFO [RS:0;fda3efb342d8:45455 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T04:27:00,158 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda3efb342d8,45455,1731731034087] 2024-11-16T04:27:00,182 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda3efb342d8,45455,1731731034087 already deleted, retry=false 2024-11-16T04:27:00,182 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda3efb342d8,45455,1731731034087 expired; onlineServers=0 2024-11-16T04:27:00,182 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fda3efb342d8,45087,1731731033310' ***** 2024-11-16T04:27:00,182 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T04:27:00,182 INFO [M:0;fda3efb342d8:45087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T04:27:00,183 INFO [M:0;fda3efb342d8:45087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T04:27:00,183 DEBUG [M:0;fda3efb342d8:45087 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T04:27:00,183 DEBUG [M:0;fda3efb342d8:45087 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T04:27:00,183 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T04:27:00,183 DEBUG [master/fda3efb342d8:0:becomeActiveMaster-HFileCleaner.large.0-1731731036975 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda3efb342d8:0:becomeActiveMaster-HFileCleaner.large.0-1731731036975,5,FailOnTimeoutGroup] 2024-11-16T04:27:00,183 DEBUG [master/fda3efb342d8:0:becomeActiveMaster-HFileCleaner.small.0-1731731036976 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda3efb342d8:0:becomeActiveMaster-HFileCleaner.small.0-1731731036976,5,FailOnTimeoutGroup] 2024-11-16T04:27:00,183 INFO [M:0;fda3efb342d8:45087 {}] hbase.ChoreService(370): Chore service for: master/fda3efb342d8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T04:27:00,183 INFO [M:0;fda3efb342d8:45087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T04:27:00,183 DEBUG [M:0;fda3efb342d8:45087 {}] master.HMaster(1795): Stopping service threads 2024-11-16T04:27:00,183 INFO [M:0;fda3efb342d8:45087 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T04:27:00,183 INFO [M:0;fda3efb342d8:45087 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T04:27:00,183 ERROR [M:0;fda3efb342d8:45087 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T04:27:00,184 INFO [M:0;fda3efb342d8:45087 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T04:27:00,184 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T04:27:00,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T04:27:00,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T04:27:00,193 DEBUG [M:0;fda3efb342d8:45087 {}] zookeeper.ZKUtil(347): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T04:27:00,193 WARN [M:0;fda3efb342d8:45087 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T04:27:00,193 INFO [M:0;fda3efb342d8:45087 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/.lastflushedseqids 2024-11-16T04:27:00,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742458_1634 (size=72) 2024-11-16T04:27:00,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T04:27:00,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45455-0x10141e284fa0001, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T04:27:00,259 INFO [RS:0;fda3efb342d8:45455 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T04:27:00,259 INFO [RS:0;fda3efb342d8:45455 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda3efb342d8,45455,1731731034087; zookeeper connection closed. 2024-11-16T04:27:00,259 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@681324ef {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@681324ef 2024-11-16T04:27:00,260 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T04:27:00,603 INFO [M:0;fda3efb342d8:45087 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T04:27:00,603 INFO [M:0;fda3efb342d8:45087 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T04:27:00,603 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T04:27:00,603 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:27:00,603 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:27:00,603 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T04:27:00,603 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:27:00,604 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=754.68 KB heapSize=927.59 KB 2024-11-16T04:27:00,617 DEBUG [M:0;fda3efb342d8:45087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b42dfef2cdb46c1af6f96c6a7ca92ff is 82, key is hbase:meta,,1/info:regioninfo/1731731038006/Put/seqid=0 2024-11-16T04:27:00,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742459_1635 (size=5672) 2024-11-16T04:27:01,021 INFO [M:0;fda3efb342d8:45087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2134 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b42dfef2cdb46c1af6f96c6a7ca92ff 2024-11-16T04:27:01,040 DEBUG [M:0;fda3efb342d8:45087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7360629700a4c6d80bc941ece0b86e4 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x8F/proc:d/1731731193762/Put/seqid=0 2024-11-16T04:27:01,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742460_1636 (size=44198) 2024-11-16T04:27:01,446 INFO [M:0;fda3efb342d8:45087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754.13 KB at sequenceid=2134 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7360629700a4c6d80bc941ece0b86e4 2024-11-16T04:27:01,449 INFO [M:0;fda3efb342d8:45087 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a7360629700a4c6d80bc941ece0b86e4 2024-11-16T04:27:01,467 DEBUG [M:0;fda3efb342d8:45087 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe07ee0ba3e146108f033ed5ef2e3fba is 69, key is fda3efb342d8,45455,1731731034087/rs:state/1731731037031/Put/seqid=0 2024-11-16T04:27:01,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073742461_1637 (size=5156) 2024-11-16T04:27:01,870 INFO [M:0;fda3efb342d8:45087 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2134 (bloomFilter=true), to=hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe07ee0ba3e146108f033ed5ef2e3fba 2024-11-16T04:27:01,873 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1b42dfef2cdb46c1af6f96c6a7ca92ff as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b42dfef2cdb46c1af6f96c6a7ca92ff 2024-11-16T04:27:01,876 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1b42dfef2cdb46c1af6f96c6a7ca92ff, entries=8, sequenceid=2134, filesize=5.5 K 2024-11-16T04:27:01,876 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a7360629700a4c6d80bc941ece0b86e4 as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a7360629700a4c6d80bc941ece0b86e4 2024-11-16T04:27:01,878 INFO [M:0;fda3efb342d8:45087 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a7360629700a4c6d80bc941ece0b86e4 2024-11-16T04:27:01,878 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a7360629700a4c6d80bc941ece0b86e4, entries=170, sequenceid=2134, filesize=43.2 K 2024-11-16T04:27:01,879 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fe07ee0ba3e146108f033ed5ef2e3fba as hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fe07ee0ba3e146108f033ed5ef2e3fba 2024-11-16T04:27:01,882 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43469/user/jenkins/test-data/599ec342-c207-bc23-3a6d-532a118d54bd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fe07ee0ba3e146108f033ed5ef2e3fba, entries=1, sequenceid=2134, filesize=5.0 K 2024-11-16T04:27:01,882 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(3140): Finished flush of dataSize ~754.68 KB/772793, heapSize ~927.30 KB/949552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2134, compaction requested=false 2024-11-16T04:27:01,884 INFO [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T04:27:01,884 DEBUG [M:0;fda3efb342d8:45087 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731731220603Disabling compacts and flushes for region at 1731731220603Disabling writes for close at 1731731220603Obtaining lock to block concurrent updates at 1731731220604 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731731220604Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=772793, getHeapSize=949792, getOffHeapSize=0, getCellsCount=2329 at 1731731220604Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731731220604Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731731220604Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731731220617 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731731220617Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731731221024 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731731221040 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731731221040Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731731221449 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731731221467 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731731221467Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b7a8e7: reopening flushed file at 1731731221873 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5262b171: reopening flushed file at 1731731221876 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b0ad443: reopening flushed file at 1731731221878 (+2 ms)Finished flush of dataSize ~754.68 KB/772793, heapSize ~927.30 KB/949552, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1279ms, sequenceid=2134, compaction requested=false at 1731731221882 (+4 ms)Writing region close event to WAL at 1731731221884 (+2 ms)Closed at 1731731221884 2024-11-16T04:27:01,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46103 is added to blk_1073741830_1006 (size=912702) 2024-11-16T04:27:01,887 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T04:27:01,887 INFO [M:0;fda3efb342d8:45087 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T04:27:01,887 INFO [M:0;fda3efb342d8:45087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45087 2024-11-16T04:27:01,887 INFO [M:0;fda3efb342d8:45087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T04:27:02,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T04:27:02,008 INFO [M:0;fda3efb342d8:45087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T04:27:02,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45087-0x10141e284fa0000, quorum=127.0.0.1:60710, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T04:27:02,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3b60cd11{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T04:27:02,016 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286a1f36{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T04:27:02,016 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T04:27:02,016 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1118b2a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T04:27:02,016 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b40f5c8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/hadoop.log.dir/,STOPPED} 2024-11-16T04:27:02,019 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T04:27:02,019 WARN [BP-685687939-172.17.0.2-1731731029115 heartbeating to localhost/127.0.0.1:43469 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T04:27:02,019 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T04:27:02,019 WARN [BP-685687939-172.17.0.2-1731731029115 heartbeating to localhost/127.0.0.1:43469 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-685687939-172.17.0.2-1731731029115 (Datanode Uuid a61d9fca-c2f0-4b33-9fbc-13316ed550d7) service to localhost/127.0.0.1:43469 2024-11-16T04:27:02,021 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/data/data1/current/BP-685687939-172.17.0.2-1731731029115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T04:27:02,021 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/cluster_8c14a4dd-cd90-ffad-25d6-331ec3d06675/data/data2/current/BP-685687939-172.17.0.2-1731731029115 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T04:27:02,021 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T04:27:02,029 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f7463d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T04:27:02,030 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T04:27:02,030 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T04:27:02,030 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T04:27:02,030 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/b871228c-1798-6ec0-26a6-ca9d9f0a6c18/hadoop.log.dir/,STOPPED} 2024-11-16T04:27:02,051 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T04:27:02,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down